add edge rendering for app dir for Turbopack (#50830)

### What?

* allow to use `runtime = "edge"` for app dir in turbopack
* move common imports from next-app-loader to
`packages/next/src/server/app-render/entry-base.ts`
* move common turbopack code to communicate between JS and Rust into
separate files

### Why?

A lot test cases depend on edge rendering

### How?

---------

Co-authored-by: Alex Kirszenberg <alex.kirszenberg@vercel.com>

packages/next-swc/crates/next-core/js/src/entry/app-edge-renderer.tsx

@@ -0,0 +1,82 @@
+// IPC need to be the first import to allow it to catch errors happening during
+// the other imports
+import startOperationStreamHandler from '../internal/operation-stream'
+
+import { join } from 'path'
+import { parse as parseUrl } from 'node:url'
+
+import { runEdgeFunction } from '../internal/edge'
+import { headersFromEntries, initProxiedHeaders } from '../internal/headers'
+import { NodeNextRequest } from 'next/dist/server/base-http/node'
+
+import type { IncomingMessage } from 'node:http'
+import type { RenderData } from 'types/turbopack'
+
+import chunkGroup from 'INNER_EDGE_CHUNK_GROUP'
+import { attachRequestMeta } from '../internal/next-request-helpers'
+import { Readable } from 'stream'
+
+startOperationStreamHandler(async (renderData: RenderData, respond) => {
+  const { response } = await runOperation(renderData)
+
+  if (response == null) {
+    throw new Error('no html returned')
+  }
+
+  const channel = respond({
+    status: response.status,
+    // @ts-expect-error Headers is iterable since node.js 18
+    headers: [...response.headers],
+  })
+
+  if (response.body) {
+    const reader = response.body.getReader()
+    for (;;) {
+      let { done, value } = await reader.read()
+      if (done) {
+        break
+      }
+      channel.chunk(Buffer.from(value!))
+    }
+  }
+
+  channel.end()
+})
+
+async function runOperation(renderData: RenderData) {
+  const edgeInfo = {
+    name: 'edge',
+    paths: chunkGroup.map((chunk: string) =>
+      join(process.cwd(), '.next/server/app', chunk)
+    ),
+    wasm: [],
+    env: Object.keys(process.env),
+    assets: [],
+  }
+
+  const parsedUrl = parseUrl(renderData.originalUrl, true)
+  const incoming = new Readable() as IncomingMessage
+  incoming.push(null)
+  incoming.url = renderData.originalUrl
+  incoming.method = renderData.method
+  incoming.headers = initProxiedHeaders(
+    headersFromEntries(renderData.rawHeaders),
+    renderData.data?.serverInfo
+  )
+  const req = new NodeNextRequest(incoming)
+  attachRequestMeta(req, parsedUrl, req.headers.host!)
+
+  const res = await runEdgeFunction({
+    edgeInfo,
+    outputDir: 'edge-pages',
+    req,
+    query: renderData.rawQuery,
+    params: renderData.params,
+    path: renderData.path,
+    onWarning(warning) {
+      console.warn(warning)
+    },
+  })
+
+  return res as { response: Response }
+}

packages/next-swc/crates/next-core/js/src/entry/app-renderer.tsx

@@ -1,22 +1,11 @@
-// Provided by the rust generate code
-type FileType =
-  | 'layout'
-  | 'template'
-  | 'error'
-  | 'loading'
-  | 'not-found'
-  | 'head'
-declare global {
-  // an tree of all layouts and the page
-  const LOADER_TREE: LoaderTree
-  // array of chunks for the bootstrap script
-  const BOOTSTRAP: string[]
-  const IPC: Ipc<unknown, unknown>
-}
+// IPC need to be the first import to allow it to catch errors happening during
+// the other imports
+import startOperationStreamHandler from '../internal/operation-stream'
+
+import '../polyfill/app-polyfills.ts'
 
-import type { Ipc } from '@vercel/turbopack-node/ipc/index'
 import type { IncomingMessage } from 'node:http'
-import type { ClientReferenceManifest } from 'next/dist/build/webpack/plugins/flight-manifest-plugin'
+
 import type { RenderData } from 'types/turbopack'
 import type { RenderOpts } from 'next/dist/server/app-render/types'
 
@@ -25,211 +14,39 @@ import { RSC_VARY_HEADER } from 'next/dist/client/components/app-router-headers'
 import { headersFromEntries, initProxiedHeaders } from '../internal/headers'
 import { parse, ParsedUrlQuery } from 'node:querystring'
 import { PassThrough } from 'node:stream'
-;('TURBOPACK { transition: next-layout-entry; chunking-type: isolatedParallel }')
-// @ts-ignore
-import layoutEntry from './app/layout-entry'
+;('TURBOPACK { chunking-type: isolatedParallel }')
+import entry from 'APP_ENTRY'
+import BOOTSTRAP from 'APP_BOOTSTRAP'
 import { createServerResponse } from '../internal/http'
+import { createManifests, installRequireAndChunkLoad } from './app/manifest'
 
-globalThis.__next_require__ = (data) => {
-  const [, , ssr_id] = JSON.parse(data)
-  return __turbopack_require__(ssr_id)
-}
-globalThis.__next_chunk_load__ = () => Promise.resolve()
+installRequireAndChunkLoad()
 
 process.env.__NEXT_NEW_LINK_BEHAVIOR = 'true'
 
-const ipc = IPC as Ipc<IpcIncomingMessage, IpcOutgoingMessage>
-
-type IpcIncomingMessage = {
-  type: 'headers'
-  data: RenderData
-}
-
-type IpcOutgoingMessage =
-  | {
-      type: 'headers'
-      data: {
-        status: number
-        headers: [string, string][]
-      }
-    }
-  | {
-      type: 'bodyChunk'
-      data: number[]
-    }
-  | {
-      type: 'bodyEnd'
-    }
-
 const MIME_TEXT_HTML_UTF8 = 'text/html; charset=utf-8'
 
-;(async () => {
-  while (true) {
-    const msg = await ipc.recv()
-
-    let renderData: RenderData
-    switch (msg.type) {
-      case 'headers': {
-        renderData = msg.data
-        break
-      }
-      default: {
-        console.error('unexpected message type', msg.type)
-        process.exit(1)
-      }
-    }
+startOperationStreamHandler(async (renderData: RenderData, respond) => {
+  const result = await runOperation(renderData)
 
-    const result = await runOperation(renderData)
-
-    if (result == null) {
-      throw new Error('no html returned')
-    }
-
-    ipc.send({
-      type: 'headers',
-      data: {
-        status: result.statusCode,
-        headers: result.headers,
-      },
-    })
-
-    for await (const chunk of result.body) {
-      ipc.send({
-        type: 'bodyChunk',
-        data: (chunk as Buffer).toJSON().data,
-      })
-    }
-
-    ipc.send({ type: 'bodyEnd' })
+  if (result == null) {
+    throw new Error('no html returned')
   }
-})().catch((err) => {
-  ipc.sendError(err)
-})
 
-// TODO expose these types in next.js
-type ComponentModule = () => any
-type ModuleReference = [componentModule: ComponentModule, filePath: string]
-export type ComponentsType = {
-  [componentKey in FileType]?: ModuleReference
-} & {
-  page?: ModuleReference
-}
-type LoaderTree = [
-  segment: string,
-  parallelRoutes: { [parallelRouterKey: string]: LoaderTree },
-  components: ComponentsType
-]
+  const channel = respond({
+    status: result.statusCode,
+    headers: result.headers,
+  })
 
-async function runOperation(renderData: RenderData) {
-  const proxyMethodsForModule = (
-    id: string
-  ): ProxyHandler<ClientReferenceManifest['ssrModuleMapping']> => {
-    return {
-      get(_target, prop: string) {
-        return {
-          id,
-          chunks: JSON.parse(id)[1],
-          name: prop,
-        }
-      },
-    }
+  for await (const chunk of result.body) {
+    channel.chunk(chunk as Buffer)
   }
 
-  const proxyMethodsNested = (
-    type: 'ssrModuleMapping' | 'clientModules' | 'entryCSSFiles'
-  ): ProxyHandler<
-    | ClientReferenceManifest['ssrModuleMapping']
-    | ClientReferenceManifest['clientModules']
-    | ClientReferenceManifest['entryCSSFiles']
-  > => {
-    return {
-      get(_target, key: string) {
-        if (type === 'ssrModuleMapping') {
-          return new Proxy({}, proxyMethodsForModule(key as string))
-        }
-        if (type === 'clientModules') {
-          // The key is a `${file}#${name}`, but `file` can contain `#` itself.
-          // There are 2 possibilities:
-          //   "file#"    => id = "file", name = ""
-          //   "file#foo" => id = "file", name = "foo"
-          const pos = key.lastIndexOf('#')
-          let id = key
-          let name = ''
-          if (pos !== -1) {
-            id = key.slice(0, pos)
-            name = key.slice(pos + 1)
-          } else {
-            throw new Error('keys need to be formatted as {file}#{name}')
-          }
-
-          return {
-            id,
-            name,
-            chunks: JSON.parse(id)[1],
-          }
-        }
-        if (type === 'entryCSSFiles') {
-          const cssChunks = JSON.parse(key)
-          // TODO(WEB-856) subscribe to changes
-          return {
-            modules: [],
-            files: cssChunks.filter(filterAvailable).map(toPath),
-          }
-        }
-      },
-    }
-  }
+  channel.end()
+})
 
-  const proxyMethods = (): ProxyHandler<ClientReferenceManifest> => {
-    const clientModulesProxy = new Proxy(
-      {},
-      proxyMethodsNested('clientModules')
-    )
-    const ssrModuleMappingProxy = new Proxy(
-      {},
-      proxyMethodsNested('ssrModuleMapping')
-    )
-    const entryCSSFilesProxy = new Proxy(
-      {},
-      proxyMethodsNested('entryCSSFiles')
-    )
-    return {
-      get(_target: any, prop: string) {
-        if (prop === 'ssrModuleMapping') {
-          return ssrModuleMappingProxy
-        }
-        if (prop === 'clientModules') {
-          return clientModulesProxy
-        }
-        if (prop === 'entryCSSFiles') {
-          return entryCSSFilesProxy
-        }
-      },
-    }
-  }
-  const availableModules = new Set()
-  const toPath = (chunk: ChunkData) =>
-    typeof chunk === 'string' ? chunk : chunk.path
-  /// determines if a chunk is needed based on the current available modules
-  const filterAvailable = (chunk: ChunkData) => {
-    if (typeof chunk === 'string') {
-      return true
-    } else {
-      let includedList = chunk.included || []
-      if (includedList.length === 0) {
-        return true
-      }
-      let needed = false
-      for (const item of includedList) {
-        if (!availableModules.has(item)) {
-          availableModules.add(item)
-          needed = true
-        }
-      }
-      return needed
-    }
-  }
-  const manifest: ClientReferenceManifest = new Proxy({} as any, proxyMethods())
+async function runOperation(renderData: RenderData) {
+  const { clientReferenceManifest } = createManifests()
 
   const req: IncomingMessage = {
     url: renderData.originalUrl,
@@ -266,12 +83,14 @@ async function runOperation(renderData: RenderData) {
       ampFirstPages: [],
     },
     ComponentMod: {
-      ...layoutEntry,
-      default: undefined,
-      tree: LOADER_TREE,
+      ...entry,
+      __next_app__: {
+        require: __next_require__,
+        loadChunk: __next_chunk_load__,
+      },
       pages: ['page.js'],
     },
-    clientReferenceManifest: manifest,
+    clientReferenceManifest,
     runtime: 'nodejs',
     serverComponents: true,
     assetPrefix: '',
@@ -305,23 +124,3 @@ async function runOperation(renderData: RenderData) {
     body,
   }
 }
-
-// This utility is based on https://github.com/zertosh/htmlescape
-// License: https://github.com/zertosh/htmlescape/blob/0527ca7156a524d256101bb310a9f970f63078ad/LICENSE
-
-const ESCAPE_LOOKUP = {
-  '&': '\\u0026',
-  '>': '\\u003e',
-  '<': '\\u003c',
-  '\u2028': '\\u2028',
-  '\u2029': '\\u2029',
-}
-
-const ESCAPE_REGEX = /[&><\u2028\u2029]/g
-
-export function htmlEscapeJsonString(str: string) {
-  return str.replace(
-    ESCAPE_REGEX,
-    (match) => ESCAPE_LOOKUP[match as keyof typeof ESCAPE_LOOKUP]
-  )
-}

packages/next-swc/crates/next-core/js/src/entry/app/app-entry.tsx

@@ -0,0 +1 @@
+// This file is generated by app_source.rs