Improved bundling strategy for the server graph (#40739)

This PR changes the external module resolution to eagerly bundle
node_modules, and some specific Next.js internal modules, if on the
`WEBPACK_LAYERS.server` layer. While resolving corresponding packages,
we use the `react-server` export condition (fallbacks to default).

A follow-up PR will be adding a Next.js option to opt-out specific
packages from being bundled on the server layer.

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

packages/next-swc/crates/core/src/react_server_components.rs

@@ -395,8 +395,14 @@ pub fn server_components<C: Comments>(
             JsWord::from("client-only"),
             JsWord::from("react-dom/client"),
             JsWord::from("react-dom/server"),
+            // TODO-APP: JsWord::from("next/router"),
+            // TODO-APP: Rule out client hooks.
+        ],
+        invalid_client_imports: vec![
+            JsWord::from("server-only"),
+            // TODO-APP: Rule out server hooks such as `useCookies`, `useHeaders`,
+            // `usePreviewData`.
         ],
-        invalid_client_imports: vec![JsWord::from("server-only")],
         invalid_server_react_dom_apis: vec![
             JsWord::from("findDOMNode"),
             JsWord::from("flushSync"),

packages/next/build/webpack-config.ts

@@ -831,6 +831,14 @@ export default async function getBaseWebpackConfig(
     [COMPILER_NAMES.edgeServer]: ['browser', 'module', 'main'],
   }
 
+  const reactAliases = {
+    react: reactDir,
+    'react-dom$': reactDomDir,
+    'react-dom/server$': `${reactDomDir}/server`,
+    'react-dom/server.browser$': `${reactDomDir}/server.browser`,
+    'react-dom/client$': `${reactDomDir}/client`,
+  }
+
   const resolveConfig = {
     // Disable .mjs for node_modules bundling
     extensions: isNodeServer
@@ -843,11 +851,8 @@ export default async function getBaseWebpackConfig(
     alias: {
       next: NEXT_PROJECT_ROOT,
 
-      react: `${reactDir}`,
-      'react-dom$': `${reactDomDir}`,
-      'react-dom/server$': `${reactDomDir}/server`,
-      'react-dom/server.browser$': `${reactDomDir}/server.browser`,
-      'react-dom/client$': `${reactDomDir}/client`,
+      ...reactAliases,
+
       'styled-jsx/style$': require.resolve(`styled-jsx/style`),
       'styled-jsx$': require.resolve(`styled-jsx`),
 
@@ -977,6 +982,7 @@ export default async function getBaseWebpackConfig(
     context: string,
     request: string,
     dependencyType: string,
+    layer: string | null,
     getResolve: (
       options: any
     ) => (
@@ -1001,14 +1007,47 @@ export default async function getBaseWebpackConfig(
       return `commonjs next/dist/lib/import-next-warning`
     }
 
+    const resolveWithReactServerCondition =
+      layer === WEBPACK_LAYERS.server
+        ? getResolve({
+            // If React is aliased to another channel during Next.js' local development,
+            // we need to provide that alias to webpack's resolver.
+            alias: process.env.__NEXT_REACT_CHANNEL
+              ? {
+                  ...reactAliases,
+                  'react/package.json': `${reactDir}/package.json`,
+                  'react/jsx-runtime': `${reactDir}/jsx-runtime`,
+                  'react/jsx-dev-runtime': `${reactDir}/jsx-dev-runtime`,
+                  'react-dom/package.json': `${reactDomDir}/package.json`,
+                }
+              : false,
+            conditionNames: ['react-server'],
+          })
+        : null
+
+    // Special internal modules that must be bundled for Server Components.
+    if (layer === WEBPACK_LAYERS.server) {
+      if (!isLocal && /^react(?:$|\/)/.test(request)) {
+        const [resolved] = await resolveWithReactServerCondition!(
+          context,
+          request
+        )
+        return resolved
+      }
+      if (
+        request ===
+        'next/dist/compiled/react-server-dom-webpack/writer.browser.server'
+      ) {
+        return
+      }
+    }
+
     // Relative requires don't need custom resolution, because they
     // are relative to requests we've already resolved here.
     // Absolute requires (require('/foo')) are extremely uncommon, but
     // also have no need for customization as they're already resolved.
     if (!isLocal) {
-      // styled-jsx is also marked as externals here to avoid being
-      // bundled in client components for RSC.
-      if (/^(?:next$|styled-jsx$|react(?:$|\/))/.test(request)) {
+      if (/^(?:next$|react(?:$|\/))/.test(request)) {
         return `commonjs ${request}`
       }
 
@@ -1122,9 +1161,22 @@ export default async function getBaseWebpackConfig(
       return
     }
 
-    // Anything else that is standard JavaScript within `node_modules`
-    // can be externalized.
     if (/node_modules[/\\].*\.[mc]?js$/.test(res)) {
+      if (layer === WEBPACK_LAYERS.server) {
+        try {
+          const [resolved] = await resolveWithReactServerCondition!(
+            context,
+            request
+          )
+          return resolved
+        } catch (err) {
+          // The `react-server` condition is not matched, fallback.
+          return
+        }
+      }
+
+      // Anything else that is standard JavaScript within `node_modules`
+      // can be externalized.
       return `${externalType} ${request}`
     }
 
@@ -1176,11 +1228,17 @@ export default async function getBaseWebpackConfig(
               context,
               request,
               dependencyType,
+              contextInfo,
               getResolve,
             }: {
               context: string
               request: string
               dependencyType: string
+              contextInfo: {
+                issuer: string
+                issuerLayer: string | null
+                compiler: string
+              }
               getResolve: (
                 options: any
               ) => (
@@ -1193,24 +1251,31 @@ export default async function getBaseWebpackConfig(
                 ) => void
               ) => void
             }) =>
-              handleExternals(context, request, dependencyType, (options) => {
-                const resolveFunction = getResolve(options)
-                return (resolveContext: string, requestToResolve: string) =>
-                  new Promise((resolve, reject) => {
-                    resolveFunction(
-                      resolveContext,
-                      requestToResolve,
-                      (err, result, resolveData) => {
-                        if (err) return reject(err)
-                        if (!result) return resolve([null, false])
-                        const isEsm = /\.js$/i.test(result)
-                          ? resolveData?.descriptionFileData?.type === 'module'
-                          : /\.mjs$/i.test(result)
-                        resolve([result, isEsm])
-                      }
-                    )
-                  })
-              }),
+              handleExternals(
+                context,
+                request,
+                dependencyType,
+                contextInfo.issuerLayer,
+                (options) => {
+                  const resolveFunction = getResolve(options)
+                  return (resolveContext: string, requestToResolve: string) =>
+                    new Promise((resolve, reject) => {
+                      resolveFunction(
+                        resolveContext,
+                        requestToResolve,
+                        (err, result, resolveData) => {
+                          if (err) return reject(err)
+                          if (!result) return resolve([null, false])
+                          const isEsm = /\.js$/i.test(result)
+                            ? resolveData?.descriptionFileData?.type ===
+                              'module'
+                            : /\.mjs$/i.test(result)
+                          resolve([result, isEsm])
+                        }
+                      )
+                    })
+                }
+              ),
           ]
         : [
             // When the 'serverless' target is used all node_modules will be compiled into the output bundles

packages/next/build/webpack/loaders/next-app-loader.ts

@@ -191,6 +191,9 @@ const nextAppLoader: webpack.LoaderDefinitionFunction<{
         : 'null'
     }
 
+    export const serverHooks = require('next/dist/client/components/hooks-server-context.js')
+
+    export const renderToReadableStream = require('next/dist/compiled/react-server-dom-webpack/writer.browser.server').renderToReadableStream
     export const __next_app_webpack_require__ = __webpack_require__
   `
 

packages/next/build/webpack/plugins/flight-client-entry-plugin.ts

@@ -59,11 +59,11 @@ export class FlightClientEntryPlugin {
     )
 
     compiler.hooks.finishMake.tapPromise(PLUGIN_NAME, (compilation) => {
-      return this.createClientEndpoints(compiler, compilation)
+      return this.createClientEntries(compiler, compilation)
     })
   }
 
-  async createClientEndpoints(compiler: any, compilation: any) {
+  async createClientEntries(compiler: any, compilation: any) {
     const promises: Array<
       ReturnType<typeof this.injectClientEntryAndSSRModules>
     > = []

packages/next/client/components/hooks-server-context.ts

@@ -1,6 +1,13 @@
 // @ts-expect-error createServerContext exists on experimental channel
 import { createServerContext } from 'react'
 
+// createServerContext exists in react@experimental + react-dom@experimental
+if (typeof createServerContext === 'undefined') {
+  throw new Error(
+    '"app" directory requires React.createServerContext which is not available in the version of React you are using. Please update to react@experimental and react-dom@experimental.'
+  )
+}
+
 export class DynamicServerError extends Error {
   constructor(type: string) {
     super(`Dynamic server usage: ${type}`)

packages/next/client/components/hooks-server.ts

@@ -1,28 +1,11 @@
-import type { AsyncLocalStorage } from 'async_hooks'
 import { useContext } from 'react'
 import {
   HeadersContext,
   PreviewDataContext,
   CookiesContext,
   DynamicServerError,
 } from './hooks-server-context'
-
-export interface StaticGenerationStore {
-  inUse?: boolean
-  pathname?: string
-  revalidate?: number
-  fetchRevalidate?: number
-  isStaticGeneration?: boolean
-}
-
-export let staticGenerationAsyncStorage:
-  | AsyncLocalStorage<StaticGenerationStore>
-  | StaticGenerationStore = {}
-
-if (process.env.NEXT_RUNTIME !== 'edge' && typeof window === 'undefined') {
-  staticGenerationAsyncStorage =
-    new (require('async_hooks').AsyncLocalStorage)()
-}
+import { staticGenerationAsyncStorage } from './static-generation-async-storage'
 
 function useStaticGenerationBailout(reason: string) {
   const staticGenerationStore =

packages/next/client/components/static-generation-async-storage.ts

@@ -0,0 +1,18 @@
+import type { AsyncLocalStorage } from 'async_hooks'
+
+export interface StaticGenerationStore {
+  inUse?: boolean
+  pathname?: string
+  revalidate?: number
+  fetchRevalidate?: number
+  isStaticGeneration?: boolean
+}
+
+export let staticGenerationAsyncStorage:
+  | AsyncLocalStorage<StaticGenerationStore>
+  | StaticGenerationStore = {}
+
+if (process.env.NEXT_RUNTIME !== 'edge' && typeof window === 'undefined') {
+  staticGenerationAsyncStorage =
+    new (require('async_hooks').AsyncLocalStorage)()
+}

packages/next/export/worker.ts

@@ -388,9 +388,7 @@ export default async function exportPage({
         // and bail when dynamic dependencies are detected
         // only fully static paths are fully generated here
         if (isAppDir) {
-          const {
-            DynamicServerError,
-          } = require('../client/components/hooks-server-context')
+          const { DynamicServerError } = components.ComponentMod.serverHooks
 
           const { renderToHTMLOrFlight } =
             require('../server/app-render') as typeof import('../server/app-render')