Next Build Turbo POC (2) (#51546)

Another attempt at getting #49942 in.

This time, the mold install step is gated to Linux.

.github/workflows/build_and_deploy.yml

@@ -98,6 +98,8 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
             build: >-
               set -e &&
+              apt update &&
+              apt install -y pkg-config &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-gnu &&
@@ -110,7 +112,7 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
-              apk add --no-cache libc6-compat &&
+              apk add --no-cache libc6-compat pkgconfig &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
               rustup target add x86_64-unknown-linux-musl &&
@@ -133,6 +135,8 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
             build: >-
               set -e &&
+              apt update &&
+              apt install -y pkg-config &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&
               rustup default "${RUST_TOOLCHAIN}" &&
@@ -146,7 +150,7 @@ jobs:
             docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
             build: >-
               set -e &&
-              apk add --no-cache libc6-compat &&
+              apk add --no-cache libc6-compat pkgconfig &&
               export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
               npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
               rustup toolchain install "${RUST_TOOLCHAIN}" &&

.github/workflows/build_and_test.yml

@@ -73,6 +73,7 @@ jobs:
       skipInstallBuild: 'yes'
       skipNativeBuild: 'yes'
       afterBuild: turbo run test-cargo-unit
+      mold: 'yes'
     secrets: inherit
 
   test-cargo-integration:

.github/workflows/build_reusable.yml

@@ -7,6 +7,12 @@ on:
         required: false
         description: 'additional steps to run'
         type: string
+      # Toggles the mold linker. The default linker is much slower and can run into OOMs.
+      # However, custom linkers won't work for wasm.
+      mold:
+        required: false
+        description: 'whether to use the mold linker'
+        type: string
       skipInstallBuild:
         required: false
         description: 'whether to skip pnpm install && pnpm build'
@@ -88,6 +94,13 @@ jobs:
         with:
           components: rustfmt, clippy
 
+      - name: 'Install mold linker'
+        if: ${{ inputs.mold == 'yes' }}
+        run: |
+          sudo apt update
+          sudo apt install -y mold
+          echo RUSTFLAGS=${RUSTFLAGS}\ -C\ link-arg=-fuse-ld=mold >> $GITHUB_ENV
+
       - name: Install nextest
         if: ${{ inputs.needsNextest == 'yes' }}
         uses: taiki-e/install-action@nextest

Cargo.toml

@@ -26,7 +26,7 @@ lto = true
 
 [workspace.dependencies]
 # Workspace crates
-next-build = { path = "packages/next-swc/crates/next-build" }
+next-build = { path = "packages/next-swc/crates/next-build", default-features = false }
 next-core = { path = "packages/next-swc/crates/next-core", default-features = false }
 next-dev = { path = "packages/next-swc/crates/next-dev", default-features = false, features = [
   "serializable",

packages/next-swc/crates/napi/Cargo.toml

@@ -48,9 +48,9 @@ turbo-tasks = { workspace = true }
 once_cell = { workspace = true }
 serde = "1"
 serde_json = "1"
-tracing = { version = "0.1.37" }
+tracing = { workspace = true }
 tracing-futures = "0.2.5"
-tracing-subscriber = "0.3.9"
+tracing-subscriber = { workspace = true }
 tracing-chrome = "0.5.0"
 turbopack-binding = { workspace = true, features = [
   "__swc_core_binding_napi",

packages/next-swc/crates/napi/src/turbopack.rs

@@ -1,7 +1,14 @@
-use std::convert::TryFrom;
+use std::{
+    convert::{TryFrom, TryInto},
+    path::PathBuf,
+};
 
+use anyhow::Context;
 use napi::bindgen_prelude::*;
-use next_build::{next_build as turbo_next_build, NextBuildOptions};
+use next_build::{
+    build as turbo_next_build, build_options::BuildContext, BuildOptions as NextBuildOptions,
+};
+use next_core::next_config::{Rewrite, Rewrites, RouteHas};
 use next_dev::{devserver_options::DevServerOptions, start_server};
 
 use crate::util::MapErr;
@@ -15,95 +22,162 @@ pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
 pub struct NextBuildContext {
+    // Added by Next.js for next build --turbo specifically.
+    /// The root directory of the workspace.
+    pub root: Option<String>,
+
+    /// The project's directory.
     pub dir: Option<String>,
-    pub app_dir: Option<String>,
-    pub pages_dir: Option<String>,
-    pub rewrites: Option<Rewrites>,
-    pub original_rewrites: Option<Rewrites>,
-    pub original_redirects: Option<Vec<Redirect>>,
+
+    /// The build ID.
+    pub build_id: Option<String>,
+
+    /// The rewrites, as computed by Next.js.
+    pub rewrites: Option<NapiRewrites>,
+    // TODO(alexkirsz) These are detected directly by Turbopack for now.
+    // pub app_dir: Option<String>,
+    // pub pages_dir: Option<String>,
+    // TODO(alexkirsz) These are used to generate route types.
+    // pub original_rewrites: Option<Rewrites>,
+    // pub original_redirects: Option<Vec<Redirect>>,
 }
 
-#[napi(object, object_to_js = false)]
-#[derive(Debug)]
-pub struct Rewrites {
-    pub fallback: Vec<Rewrite>,
-    pub after_files: Vec<Rewrite>,
-    pub before_files: Vec<Rewrite>,
+impl TryFrom<NextBuildContext> for NextBuildOptions {
+    type Error = napi::Error;
+
+    fn try_from(value: NextBuildContext) -> Result<Self> {
+        Ok(Self {
+            dir: value.dir.map(PathBuf::try_from).transpose()?,
+            root: value.root.map(PathBuf::try_from).transpose()?,
+            log_level: None,
+            show_all: true,
+            log_detail: true,
+            full_stats: true,
+            memory_limit: None,
+            build_context: Some(BuildContext {
+                build_id: value
+                    .build_id
+                    .context("NextBuildContext must provide a build ID")?,
+                rewrites: value
+                    .rewrites
+                    .context("NextBuildContext must provide rewrites")?
+                    .into(),
+            }),
+        })
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::Rewrites`]
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
-pub struct Rewrite {
-    pub source: String,
-    pub destination: String,
+pub struct NapiRewrites {
+    pub fallback: Vec<NapiRewrite>,
+    pub after_files: Vec<NapiRewrite>,
+    pub before_files: Vec<NapiRewrite>,
+}
+
+impl From<NapiRewrites> for Rewrites {
+    fn from(val: NapiRewrites) -> Self {
+        Rewrites {
+            fallback: val
+                .fallback
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+            after_files: val
+                .after_files
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+            before_files: val
+                .before_files
+                .into_iter()
+                .map(|rewrite| rewrite.into())
+                .collect(),
+        }
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::Rewrite`]
 #[napi(object, object_to_js = false)]
 #[derive(Debug)]
-pub struct Redirect {
+pub struct NapiRewrite {
     pub source: String,
     pub destination: String,
-    pub permanent: Option<bool>,
-    pub status_code: Option<u32>,
-    pub has: Option<RouteHas>,
-    pub missing: Option<RouteHas>,
+    pub base_path: Option<bool>,
+    pub locale: Option<bool>,
+    pub has: Option<Vec<NapiRouteHas>>,
+    pub missing: Option<Vec<NapiRouteHas>>,
 }
 
-#[derive(Debug)]
-pub struct RouteHas {
-    pub r#type: RouteType,
-    pub key: Option<String>,
-    pub value: Option<String>,
+impl From<NapiRewrite> for Rewrite {
+    fn from(val: NapiRewrite) -> Self {
+        Rewrite {
+            source: val.source,
+            destination: val.destination,
+            base_path: val.base_path,
+            locale: val.locale,
+            has: val
+                .has
+                .map(|has| has.into_iter().map(|has| has.into()).collect()),
+            missing: val
+                .missing
+                .map(|missing| missing.into_iter().map(|missing| missing.into()).collect()),
+        }
+    }
 }
 
+/// Keep in sync with [`next_core::next_config::RouteHas`]
 #[derive(Debug)]
-pub enum RouteType {
-    Header,
-    Query,
-    Cookie,
-    Host,
+pub enum NapiRouteHas {
+    Header { key: String, value: Option<String> },
+    Query { key: String, value: Option<String> },
+    Cookie { key: String, value: Option<String> },
+    Host { value: String },
 }
 
-impl TryFrom<String> for RouteType {
-    type Error = napi::Error;
-
-    fn try_from(value: String) -> Result<Self> {
-        match value.as_str() {
-            "header" => Ok(RouteType::Header),
-            "query" => Ok(RouteType::Query),
-            "cookie" => Ok(RouteType::Cookie),
-            "host" => Ok(RouteType::Host),
-            _ => Err(napi::Error::new(
-                napi::Status::InvalidArg,
-                "Invalid route type",
-            )),
-        }
-    }
-}
-
-impl FromNapiValue for RouteHas {
+impl FromNapiValue for NapiRouteHas {
     unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
         let object = Object::from_napi_value(env, napi_val)?;
-        let r#type = object.get_named_property::<String>("type")?;
-        Ok(RouteHas {
-            r#type: RouteType::try_from(r#type)?,
-            key: object.get("key")?,
-            value: object.get("value")?,
+        let type_ = object.get_named_property::<String>("type")?;
+        Ok(match type_.as_str() {
+            "header" => NapiRouteHas::Header {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "query" => NapiRouteHas::Query {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "cookie" => NapiRouteHas::Cookie {
+                key: object.get_named_property("key")?,
+                value: object.get_named_property("value")?,
+            },
+            "host" => NapiRouteHas::Host {
+                value: object.get_named_property("value")?,
+            },
+            _ => {
+                return Err(napi::Error::new(
+                    Status::GenericFailure,
+                    format!("invalid type for RouteHas: {}", type_),
+                ))
+            }
         })
     }
 }
 
-impl From<NextBuildContext> for NextBuildOptions {
-    fn from(value: NextBuildContext) -> Self {
-        Self {
-            dir: value.dir,
-            memory_limit: None,
-            full_stats: None,
+impl From<NapiRouteHas> for RouteHas {
+    fn from(val: NapiRouteHas) -> Self {
+        match val {
+            NapiRouteHas::Header { key, value } => RouteHas::Header { key, value },
+            NapiRouteHas::Query { key, value } => RouteHas::Query { key, value },
+            NapiRouteHas::Cookie { key, value } => RouteHas::Cookie { key, value },
+            NapiRouteHas::Host { value } => RouteHas::Host { value },
         }
     }
 }
 
 #[napi]
 pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
-    turbo_next_build(ctx.into()).await.convert_err()
+    turbo_next_build(ctx.try_into()?).await.convert_err()
 }