app dir fixes (#48025)

### What?

* fix app dir chunking
* fix app dir 404s
* improve app dir performance
* rerender shadowportal on errors to re-add nextjs-portal to avoid empty
page
* inject polyfills before user code
* fix manifest generation

see also vercel/turbo#4488

### Why?

App dir was very slow and lead to 404 errors

### How?

add included metadata to chunks to allow deduplicate chunk loads at
runtime

packages/next-swc/Cargo.toml

@@ -47,11 +47,11 @@ swc_emotion = { version = "0.29.10" }
 testing = { version = "0.31.31" }
 
 # Turbo crates
-turbo-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230414.4" }
+turbo-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230417.2" }
 # [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
-turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230414.4" }
+turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230417.2" }
 # [TODO]: need to refactor embed_directory! macro usage in next-core
-turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230414.4" }
+turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230417.2" }
 
 # General Deps
 

packages/next-swc/crates/next-core/js/package.json

@@ -10,8 +10,8 @@
     "check": "tsc --noEmit"
   },
   "dependencies": {
-    "@vercel/turbopack-dev": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230414.4",
-    "@vercel/turbopack-node": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230414.4",
+    "@vercel/turbopack-dev": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230417.2",
+    "@vercel/turbopack-node": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-node/js?turbopack-230417.2",
     "anser": "^2.1.1",
     "css.escape": "^1.5.1",
     "next": "*",

packages/next-swc/crates/next-core/js/src/entry/app-renderer.tsx

@@ -23,9 +23,6 @@ import type {
 import type { RenderData } from 'types/turbopack'
 import type { RenderOpts } from 'next/dist/server/app-render/types'
 
-import 'next/dist/server/node-polyfill-fetch'
-import 'next/dist/server/node-polyfill-web-streams'
-import '../polyfill/async-local-storage'
 import { renderToHTMLOrFlight } from 'next/dist/server/app-render/app-render'
 import { RSC_VARY_HEADER } from 'next/dist/client/components/app-router-headers'
 import { ServerResponseShim } from '../internal/http'
@@ -127,7 +124,6 @@ type LoaderTree = [
 ]
 
 async function runOperation(renderData: RenderData) {
-  const layoutInfoChunks: Record<string, string[]> = {}
   let tree: LoaderTree = LOADER_TREE
 
   const proxyMethodsForModule = (
@@ -167,7 +163,7 @@ async function runOperation(renderData: RenderData) {
             id = key.slice(0, pos)
             name = key.slice(pos + 1)
           } else {
-            throw new Error('key need to be in format of ${file}#${name}')
+            throw new Error('keys need to be formatted as {file}#{name}')
           }
 
           return {
@@ -203,19 +199,45 @@ async function runOperation(renderData: RenderData) {
       },
     }
   }
+  const availableModules = new Set()
+  const toPath = (chunk: ChunkData) =>
+    typeof chunk === 'string' ? chunk : chunk.path
+  /// determines if a chunk is needed based on the current available modules
+  const filterAvailable = (chunk: ChunkData) => {
+    if (typeof chunk === 'string') {
+      return true
+    } else {
+      let includedList = chunk.included || []
+      if (includedList.length === 0) {
+        return true
+      }
+      let needed = false
+      for (const item of includedList) {
+        if (!availableModules.has(item)) {
+          availableModules.add(item)
+          needed = true
+        }
+      }
+      return needed
+    }
+  }
   const cssFilesProxyMethods = {
     get(_target: any, prop: string) {
-      const chunks = JSON.parse(prop)
-      const cssChunks = chunks.filter((path: string) => path.endsWith('.css'))
-      return cssChunks
+      const cssChunks = JSON.parse(prop)
+      // TODO(WEB-856) subscribe to changes
+      return cssChunks.map(toPath)
     },
   }
   const cssImportProxyMethods = {
     get(_target: any, prop: string) {
-      const chunks = JSON.parse(prop.replace(/\.js$/, ''))
+      const cssChunks = JSON.parse(prop.replace(/\.js$/, ''))
+      // TODO(WEB-856) subscribe to changes
 
-      const cssChunks = chunks.filter((path: string) => path.endsWith('.css'))
-      return cssChunks.map((chunk: string) => JSON.stringify([chunk, [chunk]]))
+      // This return value is passed to proxyMethodsNested for clientModules
+      return cssChunks
+        .filter(filterAvailable)
+        .map(toPath)
+        .map((chunk: string) => JSON.stringify([chunk, [chunk]]))
     },
   }
   const manifest: ClientReferenceManifest = new Proxy({} as any, proxyMethods())
@@ -241,10 +263,7 @@ async function runOperation(renderData: RenderData) {
     dev: true,
     buildManifest: {
       polyfillFiles: [],
-      rootMainFiles: Object.values(layoutInfoChunks)
-        .flat()
-        .concat(BOOTSTRAP)
-        .filter((path) => path.endsWith('.js')),
+      rootMainFiles: BOOTSTRAP.filter((path) => path.endsWith('.js')),
       devFiles: [],
       ampDevFiles: [],
       lowPriorityFiles: [],