Next Build Turbo POC (#49942)

This contains the original POC for `next build --turbo`. The implementation is _just enough_ to get pages building, and doesn't support the app router yet.

I'll write more details here on the implementation and what the next steps are next week.

Necessary changes on the Turbo side: vercel/turbo#4998

.cargo/config.toml

@@ -5,6 +5,10 @@ CARGO_WORKSPACE_DIR = { value = "", relative = true }
 
 rustdocflags = []
 
+[target.x86_64-unknown-linux-gnu]
+# Should be kept in sync with turbopack's linker
+rustflags = ["-C", "link-arg=-fuse-ld=mold"]
+
 [target.x86_64-pc-windows-msvc]
 linker = "rust-lld"
 

.github/actions/setup-rust/action.yml

@@ -52,6 +52,12 @@ runs:
           echo CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse >> $GITHUB_ENV
         fi
 
+    - shell: bash
+      run: |
+        : install mold linker
+        sudo apt update
+        sudo apt install -y mold
+
     - name: 'Setup Rust toolchain'
       uses: dtolnay/rust-toolchain@master
       if: ${{ !inputs.skip-install }}

packages/next-swc/crates/napi/Cargo.toml

@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true }
 once_cell = { workspace = true }
 serde = "1"
 serde_json = "1"
-tracing = { version = "0.1.37" }
+tracing = { workspace = true }
 tracing-futures = "0.2.5"
-tracing-subscriber = "0.3.9"
+tracing-subscriber = { workspace = true }
 tracing-chrome = "0.5.0"
 turbopack-binding = { workspace = true, features = [
   "__swc_core_binding_napi",

packages/next-swc/crates/napi/src/turbopack.rs

@@ -1,7 +1,14 @@
-use std::convert::TryFrom;
+use std::{
+    convert::{TryFrom, TryInto},
+    path::PathBuf,
+};
 
+use anyhow::Context;
 use napi::bindgen_prelude::*;
-use next_build::{next_build as turbo_next_build, NextBuildOptions};
+use next_build::{
+    build as turbo_next_build, build_options::BuildContext, BuildOptions as NextBuildOptions,
+};
+use next_core::next_config::{Rewrite, Rewrites, RouteHas};
 use next_dev::{devserver_options::DevServerOptions, start_server};
 
 use crate::util::MapErr;
@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
 pub struct NextBuildContext {
+    // Added by Next.js for next build --turbo specifically.
+    /// The root directory of the workspace.
+    pub root: Option<String>,
+
+    /// The project's directory.
     pub dir: Option<String>,
-    pub app_dir: Option<String>,
-    pub pages_dir: Option<String>,
-    pub rewrites: Option<Rewrites>,
-    pub original_rewrites: Option<Rewrites>,
-    pub original_redirects: Option<Vec<Redirect>>,
+
+    /// The build ID.
+    pub build_id: Option<String>,
+
+    /// The rewrites, as computed by Next.js.
+    pub rewrites: Option<NapiRewrites>,
+    // TODO(alexkirsz) These are detected directly by Turbopack for now.
+    // pub app_dir: Option<String>,
+    // pub pages_dir: Option<String>,
+    // TODO(alexkirsz) These are used to generate route types.
+    // pub original_rewrites: Option<Rewrites>,
+    // pub original_redirects: Option<Vec<Redirect>>,
 }
 
-#[napi(object, object_to_js = false)]
-#[derive(Debug)]
-pub struct Rewrites {
-    pub fallback: Vec<Rewrite>,
-    pub after_files: Vec<Rewrite>,
-    pub before_files: Vec<Rewrite>,
+impl TryFrom<NextBuildContext> for NextBuildOptions {
+    type Error = napi::Error;
+
+    fn try_from(value: NextBuildContext) -> Result<Self> {
+        Ok(Self {
+            dir: value.dir.map(PathBuf::try_from).transpose()?,
+            root: value.root.map(PathBuf::try_from).transpose()?,
+            log_level: None,
+            show_all: true,
+            log_detail: true,
+            full_stats: true,
+            memory_limit: None,
+            build_context: Some(BuildContext {
+                build_id: value
+                    .build_id
+                    .context("NextBuildContext must provide a build ID")?,
+                rewrites: value
+                    .rewrites
+                    .context("NextBuildContext must provide rewrites")?
+                    .into(),
+            }),
+        })
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::Rewrites`]
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
-pub struct Rewrite {
-    pub source: String,
-    pub destination: String,
+pub struct NapiRewrites {
+    pub fallback: Vec<NapiRewrite>,
+    pub after_files: Vec<NapiRewrite>,
+    pub before_files: Vec<NapiRewrite>,
+}
+
+impl From<NapiRewrites> for Rewrites {
+    fn from(val: NapiRewrites) -> Self {
+        Rewrites {
+            fallback: val
+                .fallback
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+            after_files: val
+                .after_files
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+            before_files: val
+                .before_files
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+        }
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::Rewrite`]
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
-pub struct Redirect {
+pub struct NapiRewrite {
     pub source: String,
     pub destination: String,
-    pub permanent: Option<bool>,
-    pub status_code: Option<u32>,
-    pub has: Option<RouteHas>,
-    pub missing: Option<RouteHas>,
+    pub base_path: Option<bool>,
+    pub locale: Option<bool>,
+    pub has: Option<Vec<NapiRouteHas>>,
+    pub missing: Option<Vec<NapiRouteHas>>,
 }
 
-#[derive(Debug)]
-pub struct RouteHas {
-    pub r#type: RouteType,
-    pub key: Option<String>,
-    pub value: Option<String>,
+impl From<NapiRewrite> for Rewrite {
+    fn from(val: NapiRewrite) -> Self {
+        Rewrite {
+            source: val.source,
+            destination: val.destination,
+            base_path: val.base_path,
+            locale: val.locale,
+            has: val
+                .has
+                .map(|has| has.into_iter().map(|has| has.into()).collect()),
+            missing: val
+                .missing
+                .map(|missing| missing.into_iter().map(|missing| missing.into()).collect()),
+        }
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::RouteHas`]
 #[derive(Debug)]
-pub enum RouteType {
-    Header,
-    Query,
-    Cookie,
-    Host,
+pub enum NapiRouteHas {
+    Header { key: String, value: Option<String> },
+    Query { key: String, value: Option<String> },
+    Cookie { key: String, value: Option<String> },
+    Host { value: String },
 }
 
-impl TryFrom<String> for RouteType {
-    type Error = napi::Error;
-
-    fn try_from(value: String) -> Result<Self> {
-        match value.as_str() {
-            "header" => Ok(RouteType::Header),
-            "query" => Ok(RouteType::Query),
-            "cookie" => Ok(RouteType::Cookie),
-            "host" => Ok(RouteType::Host),
-            _ => Err(napi::Error::new(
-                napi::Status::InvalidArg,
-                "Invalid route type",
-            )),
-        }
-    }
-}
-
-impl FromNapiValue for RouteHas {
+impl FromNapiValue for NapiRouteHas {
     unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
         let object = Object::from_napi_value(env, napi_val)?;
-        let r#type = object.get_named_property::<String>("type")?;
-        Ok(RouteHas {
-            r#type: RouteType::try_from(r#type)?,
-            key: object.get("key")?,
-            value: object.get("value")?,
+        let type_ = object.get_named_property::<String>("type")?;
+        Ok(match type_.as_str() {
+            "header" => NapiRouteHas::Header {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "query" => NapiRouteHas::Query {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "cookie" => NapiRouteHas::Cookie {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "host" => NapiRouteHas::Host {
+                value: object.get_named_property("value")?,
+            },
+            _ => {
+                return Err(napi::Error::new(
+                    Status::GenericFailure,
+                    format!("invalid type for RouteHas: {}", type_),
+                ))
+            }
         })
     }
 }
 
-impl From<NextBuildContext> for NextBuildOptions {
-    fn from(value: NextBuildContext) -> Self {
-        Self {
-            dir: value.dir,
-            memory_limit: None,
-            full_stats: None,
+impl From<NapiRouteHas> for RouteHas {
+    fn from(val: NapiRouteHas) -> Self {
+        match val {
+            NapiRouteHas::Header { key, value } => RouteHas::Header { key, value },
+            NapiRouteHas::Query { key, value } => RouteHas::Query { key, value },
+            NapiRouteHas::Cookie { key, value } => RouteHas::Cookie { key, value },
+            NapiRouteHas::Host { value } => RouteHas::Host { value },
         }
     }
 }
 
 #[napi]
 pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
-    turbo_next_build(ctx.into()).await.convert_err()
+    turbo_next_build(ctx.try_into()?).await.convert_err()
 }

packages/next-swc/crates/next-build/Cargo.toml

@@ -6,16 +6,67 @@ license = "MPL-2.0"
 edition = "2021"
 autobenches = false
 
+[[bin]]
+name = "next-build"
+path = "src/main.rs"
+bench = false
+required-features = ["cli"]
+
+[lib]
+bench = false
+
 [features]
+# By default, we enable native-tls for reqwest via downstream transitive features.
+# This is for the convenience of running daily dev workflows, i.e running
+# `cargo xxx` without explicitly specifying features, not that we want to
+# promote this as default backend. Actual configuration is done when building next-swc,
+# and also turbopack standalone when we have it.
+default = ["cli", "custom_allocator", "native-tls"]
+cli = ["clap"]
+tokio_console = [
+  "dep:console-subscriber",
+  "tokio/tracing",
+  "turbo-tasks/tokio_tracing",
+]
 native-tls = ["next-core/native-tls"]
 rustls-tls = ["next-core/rustls-tls"]
-custom_allocator = ["turbopack-binding/__turbo_tasks_malloc", "turbopack-binding/__turbo_tasks_malloc_custom_allocator"]
+custom_allocator = [
+  "turbopack-binding/__turbo_tasks_malloc",
+  "turbopack-binding/__turbo_tasks_malloc_custom_allocator",
+]
+serializable = []
+profile = []
 
 [dependencies]
-anyhow = "1.0.47"
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["derive", "env"], optional = true }
+console-subscriber = { workspace = true, optional = true }
+dunce = { workspace = true }
 next-core = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
 
-turbopack-binding = { workspace = true, features = ["__turbo_tasks", "__turbo_tasks_memory"] }
+turbopack-binding = { workspace = true, features = [
+  "__turbo_tasks",
+  "__turbo_tasks_malloc",
+  "__turbo_tasks_memory",
+  "__turbo_tasks_env",
+  "__turbo_tasks_fs",
+  "__turbo_tasks_memory",
+  "__turbopack",
+  "__turbopack_build",
+  "__turbopack_cli_utils",
+  "__turbopack_core",
+  "__turbopack_dev",
+  "__turbopack_ecmascript",
+  "__turbopack_ecmascript_runtime",
+  "__turbopack_env",
+  "__turbopack_node",
+] }
+turbo-tasks = { workspace = true }
 
 [build-dependencies]
 turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }