turbopack: add incremental cache for node.js app rendering

packages/next-swc/crates/next-core/js/src/entry/app-renderer.tsx

@@ -19,6 +19,9 @@ import entry from 'APP_ENTRY'
 import BOOTSTRAP from 'APP_BOOTSTRAP'
 import { createServerResponse } from '../internal/http'
 import { createManifests, installRequireAndChunkLoad } from './app/manifest'
+import { join } from 'node:path'
+import { nodeFs } from 'next/dist/server/lib/node-fs-methods'
+import { IncrementalCache } from 'next/dist/server/lib/incremental-cache'
 
 installRequireAndChunkLoad()
 
@@ -57,6 +60,8 @@ async function runOperation(renderData: RenderData) {
     ),
   } as any
 
+  const url = new URL(renderData.originalUrl, 'next://')
+
   const res = createServerResponse(req, renderData.path)
 
   const query = parse(renderData.rawQuery)
@@ -90,6 +95,32 @@ async function runOperation(renderData: RenderData) {
       },
       pages: ['page.js'],
     },
+    incrementalCache: new IncrementalCache({
+      fs: nodeFs,
+      dev: true,
+      requestHeaders: { ...req.headers },
+      requestProtocol: url.protocol.replace(/:$/, '') as 'http' | 'https',
+      appDir: true,
+      allowedRevalidateHeaderKeys: renderData.data?.allowedRevalidateHeaderKeys,
+      minimalMode: false,
+      serverDistDir: join(process.cwd(), '.next/server'),
+      fetchCache: true,
+      fetchCacheKeyPrefix: renderData.data?.fetchCacheKeyPrefix,
+      maxMemoryCacheSize: renderData.data?.isrMemoryCacheSize,
+      flushToDisk: false,
+      getPrerenderManifest: () => ({
+        version: 4,
+        routes: {},
+        dynamicRoutes: {},
+        preview: {
+          previewModeEncryptionKey: '',
+          previewModeId: '',
+          previewModeSigningKey: '',
+        },
+        notFoundRoutes: [],
+      }),
+      CurCacheHandler: undefined,
+    }),
     clientReferenceManifest,
     runtime: 'nodejs',
     serverComponents: true,

packages/next-swc/crates/next-core/js/src/overlay/internal/components/ShadowPortal.tsx

@@ -21,7 +21,11 @@ export function ShadowPortal({ children, globalOverlay }: ShadowPortalProps) {
     ownerDocument.body.appendChild(portalNode.current)
     forceUpdate({})
     return () => {
-      if (portalNode.current && portalNode.current.ownerDocument) {
+      if (
+        portalNode.current &&
+        portalNode.current.ownerDocument &&
+        portalNode.current.parentNode
+      ) {
         portalNode.current.ownerDocument.body.removeChild(portalNode.current)
       }
     }

packages/next-swc/crates/next-core/js/types/turbopack.d.ts

@@ -12,5 +12,8 @@ export type RenderData = {
   data?: {
     nextConfigOutput?: RenderOptsPartial['nextConfigOutput']
     serverInfo?: ServerInfo
+    allowedRevalidateHeaderKeys?: string[]
+    fetchCacheKeyPrefix?: string
+    isrMemoryCacheSize?: number
   }
 }

packages/next-swc/crates/next-core/src/next_config.rs

@@ -383,6 +383,10 @@ pub struct ExperimentalConfig {
     pub app_dir: Option<bool>,
     pub server_components_external_packages: Option<Vec<String>>,
     pub turbo: Option<ExperimentalTurboConfig>,
+    pub allowed_revalidate_header_keys: Option<Vec<String>>,
+    pub fetch_cache_key_prefix: Option<String>,
+    pub isr_memory_cache_size: Option<f64>,
+    pub isr_flush_to_disk: Option<bool>,
     mdx_rs: Option<bool>,
 
     // unsupported
@@ -404,8 +408,6 @@ pub struct ExperimentalConfig {
     fully_specified: Option<bool>,
     gzip_size: Option<bool>,
     incremental_cache_handler_path: Option<String>,
-    isr_flush_to_disk: Option<bool>,
-    isr_memory_cache_size: Option<f64>,
     large_page_data_bytes: Option<f64>,
     legacy_browsers: Option<bool>,
     manual_client_base_path: Option<bool>,

packages/next-swc/crates/next-core/src/util.rs

@@ -366,14 +366,24 @@ pub async fn render_data(
     struct Data {
         next_config_output: Option<OutputType>,
         server_info: Option<ServerInfo>,
+        allowed_revalidate_header_keys: Option<Vec<String>>,
+        fetch_cache_key_prefix: Option<String>,
+        isr_memory_cache_size: Option<f64>,
+        isr_flush_to_disk: Option<bool>,
     }
 
     let config = next_config.await?;
     let server_info = ServerInfo::try_from(&*server_addr.await?);
 
+    let experimental = &config.experimental;
+
     let value = serde_json::to_value(Data {
         next_config_output: config.output.clone(),
         server_info: server_info.ok(),
+        allowed_revalidate_header_keys: experimental.allowed_revalidate_header_keys.clone(),
+        fetch_cache_key_prefix: experimental.fetch_cache_key_prefix.clone(),
+        isr_memory_cache_size: experimental.isr_memory_cache_size.clone(),
+        isr_flush_to_disk: experimental.isr_flush_to_disk.clone(),
     })?;
     Ok(JsonValue(value).cell())
 }

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.readFile(__q____q____q____star__0__-79ac62.txt

@@ -0,0 +1,41 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.readFile(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 17,
+                column: 19,
+            },
+            end: SourcePos {
+                line: 17,
+                column: 19,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[next]/entry/app-renderer.tsx",
+                ),
+                description: "server-side rendering /",
+            },
+        ],
+    ),
+}

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.readFile(__q____q____q____star__0__-b1b1c4.txt

@@ -0,0 +1,35 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.readFile(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 17,
+                column: 19,
+            },
+            end: SourcePos {
+                line: 17,
+                column: 19,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+        ],
+    ),
+}

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.readFileSync(__q____q____q____star_-4ef8e2.txt

@@ -0,0 +1,41 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.readFileSync(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 18,
+                column: 23,
+            },
+            end: SourcePos {
+                line: 18,
+                column: 23,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[next]/entry/app-renderer.tsx",
+                ),
+                description: "server-side rendering /",
+            },
+        ],
+    ),
+}

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.readFileSync(__q____q____q____star_-c3fa50.txt

@@ -0,0 +1,35 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.readFileSync(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 18,
+                column: 23,
+            },
+            end: SourcePos {
+                line: 18,
+                column: 23,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+        ],
+    ),
+}

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.stat(__q____q____q____star__0__star-cc5e88.txt

@@ -0,0 +1,35 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.stat(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 23,
+                column: 15,
+            },
+            end: SourcePos {
+                line: 23,
+                column: 15,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+        ],
+    ),
+}

packages/next-swc/crates/next-dev-tests/tests/integration/next/app/404-custom/issues/lint TP1004 fs.stat(__q____q____q____star__0__star-f6a046.txt

@@ -0,0 +1,41 @@
+PlainIssue {
+    severity: Warning,
+    context: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+    category: "parse",
+    title: "lint TP1004 fs.stat(???*0*) is very dynamic",
+    description: "- *0* f\n  ⚠\u{fe0f}  pattern without value",
+    detail: "",
+    documentation_link: "",
+    source: Some(
+        PlainIssueSource {
+            asset: PlainAsset {
+                ident: "[project]/packages/next/dist/server/lib/node-fs-methods.js",
+            },
+            start: SourcePos {
+                line: 23,
+                column: 15,
+            },
+            end: SourcePos {
+                line: 23,
+                column: 15,
+            },
+        },
+    ),
+    sub_issues: [],
+    processing_path: Some(
+        [
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[project]/packages/next-swc/crates/next-dev-tests/tests/temp/next/app/404-custom/input/app",
+                ),
+                description: "Next.js App Page Route /",
+            },
+            PlainIssueProcessingPathItem {
+                context: Some(
+                    "[next]/entry/app-renderer.tsx",
+                ),
+                description: "server-side rendering /",
+            },
+        ],
+    ),
+}