feat(experimental): option to polyfill fetch using undici in Node…

….js <18 (#40318)

This PR adds a new `experimental.enableUndici` option to let the
developer switch from `next-fetch` to `undici` as the underlying
polyfill for `fetch` in Node.js.

In the current implementation, Next.js makes sure that `fetch` is always
available by using `node-fetch`. However, we do not polyfill in Node.js
18+, since those versions come with their own `fetch` implementation
already, built-in.

Node.js 18+ uses `undici` under the hood, so letting the developer use
`undici` earlier could make the migration easier later on.

Eventually, we hope to be able to stop polyfilling `fetch` in an
upcoming major version of Next.js, shipping less code.


## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The examples guidelines are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing.md#adding-examples)

Co-authored-by: Balázs Orbán <info@balazsorban.com>
Co-authored-by: Sukka <isukkaw@gmail.com>
Co-authored-by: JJ Kasper <jj@jjsweb.site>
Co-authored-by: Steven <steven@ceriously.com>

jest.config.js

@@ -3,13 +3,17 @@ const nextJest = require('next/jest')
 const createJestConfig = nextJest()
 
 // Any custom config you want to pass to Jest
+/** @type {import('jest').Config} */
 const customJestConfig = {
   testMatch: ['**/*.test.js', '**/*.test.ts', '**/*.test.tsx'],
   setupFilesAfterEnv: ['<rootDir>/jest-setup-after-env.ts'],
   verbose: true,
   rootDir: 'test',
   modulePaths: ['<rootDir>/lib'],
   transformIgnorePatterns: ['/next[/\\\\]dist/', '/\\.next/'],
+  globals: {
+    AbortSignal: global.AbortSignal,
+  },
 }
 
 // createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async

packages/next/build/index.ts

@@ -1205,6 +1205,7 @@ export default async function build(
               configFileName,
               runtimeEnvConfig,
               httpAgentOptions: config.httpAgentOptions,
+              enableUndici: config.experimental.enableUndici,
               locales: config.i18n?.locales,
               defaultLocale: config.i18n?.defaultLocale,
               pageRuntime: config.experimental.runtime,
@@ -1352,6 +1353,7 @@ export default async function build(
                           configFileName,
                           runtimeEnvConfig,
                           httpAgentOptions: config.httpAgentOptions,
+                          enableUndici: config.experimental.enableUndici,
                           locales: config.i18n?.locales,
                           defaultLocale: config.i18n?.defaultLocale,
                           parentId: isPageStaticSpan.id,

packages/next/build/utils.ts

@@ -41,7 +41,7 @@ import {
   LoadComponentsReturnType,
 } from '../server/load-components'
 import { trace } from '../trace'
-import { setHttpAgentOptions } from '../server/config'
+import { setHttpClientAndAgentOptions } from '../server/config'
 import { recursiveDelete } from '../lib/recursive-delete'
 import { Sema } from 'next/dist/compiled/async-sema'
 import { MiddlewareManifest } from './webpack/plugins/middleware-plugin'
@@ -1169,6 +1169,7 @@ export async function isPageStatic({
   configFileName,
   runtimeEnvConfig,
   httpAgentOptions,
+  enableUndici,
   locales,
   defaultLocale,
   parentId,
@@ -1184,6 +1185,7 @@ export async function isPageStatic({
   configFileName: string
   runtimeEnvConfig: any
   httpAgentOptions: NextConfigComplete['httpAgentOptions']
+  enableUndici?: NextConfigComplete['experimental']['enableUndici']
   locales?: string[]
   defaultLocale?: string
   parentId?: any
@@ -1210,7 +1212,10 @@ export async function isPageStatic({
   return isPageStaticSpan
     .traceAsyncFn(async () => {
       require('../shared/lib/runtime-config').setConfig(runtimeEnvConfig)
-      setHttpAgentOptions(httpAgentOptions)
+      setHttpClientAndAgentOptions({
+        httpAgentOptions,
+        experimental: { enableUndici },
+      })
 
       let componentsResult: LoadComponentsReturnType
       let prerenderRoutes: Array<string> | undefined

packages/next/compiled/undici/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Matteo Collina and Undici contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/next/compiled/undici/package.json

@@ -0,0 +1 @@
+{"name":"undici","main":"index.js","author":"Matteo Collina <hello@matteocollina.com>","license":"MIT"}

packages/next/export/index.ts

@@ -619,6 +619,7 @@ export default async function exportApp(
             httpAgentOptions: nextConfig.httpAgentOptions,
             serverComponents: !!nextConfig.experimental.appDir,
             appPaths: options.appPaths || [],
+            enableUndici: nextConfig.experimental.enableUndici,
           })
 
           for (const validation of result.ampValidations || []) {

packages/next/export/worker.ts

@@ -23,7 +23,7 @@ import { requireFontManifest } from '../server/require'
 import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
 import { trace } from '../trace'
 import { isInAmpMode } from '../shared/lib/amp-mode'
-import { setHttpAgentOptions } from '../server/config'
+import { setHttpClientAndAgentOptions } from '../server/config'
 import RenderResult from '../server/render-result'
 import isError from '../lib/is-error'
 import { addRequestMeta } from '../server/request-meta'
@@ -70,6 +70,7 @@ interface ExportPageInput {
   httpAgentOptions: NextConfigComplete['httpAgentOptions']
   serverComponents?: boolean
   appPaths: string[]
+  enableUndici: NextConfigComplete['experimental']['enableUndici']
 }
 
 interface ExportPageResults {
@@ -119,8 +120,12 @@ export default async function exportPage({
   disableOptimizedLoading,
   httpAgentOptions,
   serverComponents,
+  enableUndici,
 }: ExportPageInput): Promise<ExportPageResults> {
-  setHttpAgentOptions(httpAgentOptions)
+  setHttpClientAndAgentOptions({
+    httpAgentOptions,
+    experimental: { enableUndici },
+  })
   const exportPageSpan = trace('export-page-worker', parentSpanId)
 
   return exportPageSpan.traceAsyncFn(async () => {

packages/next/package.json

@@ -264,6 +264,7 @@
     "timers-browserify": "2.0.12",
     "tty-browserify": "0.0.1",
     "ua-parser-js": "0.7.28",
+    "undici": "5.10.0",
     "unistore": "3.4.1",
     "util": "0.12.4",
     "uuid": "8.3.2",

packages/next/server/config-schema.ts

@@ -390,6 +390,9 @@ const configSchema = {
           },
           type: 'array',
         },
+        enableUndici: {
+          type: 'boolean',
+        },
         workerThreads: {
           type: 'boolean',
         },

packages/next/server/config-shared.ts

@@ -146,6 +146,7 @@ export interface ExperimentalConfig {
    * [webpack/webpack#ModuleNotoundError.js#L13-L42](https://github.com/webpack/webpack/blob/2a0536cf510768111a3a6dceeb14cb79b9f59273/lib/ModuleNotFoundError.js#L13-L42)
    */
   fallbackNodePolyfills?: false
+  enableUndici?: boolean
   sri?: {
     algorithm?: SubresourceIntegrityAlgorithm
   }
@@ -585,6 +586,7 @@ export const defaultConfig: NextConfig = {
     amp: undefined,
     urlImports: undefined,
     modularizeImports: undefined,
+    enableUndici: false,
     adjustFontFallbacks: false,
   },
 }

packages/next/server/config-utils.ts

@@ -139,6 +139,7 @@ export function loadWebpackHook() {
         'next/dist/compiled/@babel/runtime/package.json',
       ],
       ['node-fetch', 'next/dist/compiled/node-fetch'],
+      ['undici', 'next/dist/compiled/undici'],
     ].map(
       // Use dynamic require.resolve to avoid statically analyzable since they're only for build time
       ([request, replacement]) => [request, require.resolve(replacement)]

packages/next/server/config.ts

@@ -13,6 +13,7 @@ import {
   ExperimentalConfig,
   NextConfigComplete,
   validateConfig,
+  NextConfig,
 } from './config-shared'
 import { loadWebpackHook } from './config-utils'
 import {
@@ -22,6 +23,7 @@ import {
 } from '../shared/lib/image-config'
 import { loadEnvConfig } from '@next/env'
 import { hasNextSupport } from '../telemetry/ci-info'
+import { gte as semverGte } from 'next/dist/compiled/semver'
 
 export { DomainLocale, NextConfig, normalizeConfig } from './config-shared'
 
@@ -45,21 +47,32 @@ const experimentalWarning = execOnce(
   }
 )
 
-export function setHttpAgentOptions(
-  options: NextConfigComplete['httpAgentOptions']
-) {
+export function setHttpClientAndAgentOptions(options: NextConfig) {
+  if (semverGte(process.version, '16.8.0')) {
+    if (semverGte(process.version, '18.0.0')) {
+      Log.warn(
+        '`enableUndici` option is unnecessary in Node.js v18.0.0 or greater.'
+      )
+    }
+    ;(global as any).__NEXT_USE_UNDICI = options.experimental?.enableUndici
+  } else if (options.experimental?.enableUndici) {
+    Log.warn(
+      '`enableUndici` option requires Node.js v16.8.0 or greater. Falling back to `node-fetch`'
+    )
+  }
   if ((global as any).__NEXT_HTTP_AGENT) {
     // We only need to assign once because we want
-    // to resuse the same agent for all requests.
+    // to reuse the same agent for all requests.
     return
   }
 
   if (!options) {
     throw new Error('Expected config.httpAgentOptions to be an object')
   }
 
-  ;(global as any).__NEXT_HTTP_AGENT = new HttpAgent(options)
-  ;(global as any).__NEXT_HTTPS_AGENT = new HttpsAgent(options)
+  ;(global as any).__NEXT_HTTP_AGENT_OPTIONS = options.httpAgentOptions
+  ;(global as any).__NEXT_HTTP_AGENT = new HttpAgent(options.httpAgentOptions)
+  ;(global as any).__NEXT_HTTPS_AGENT = new HttpsAgent(options.httpAgentOptions)
 }
 
 function assignDefaults(userConfig: { [key: string]: any }) {
@@ -545,9 +558,7 @@ function assignDefaults(userConfig: { [key: string]: any }) {
 
   // TODO: Change defaultConfig type to NextConfigComplete
   // so we don't need "!" here.
-  setHttpAgentOptions(
-    result.httpAgentOptions || defaultConfig.httpAgentOptions!
-  )
+  setHttpClientAndAgentOptions(result || defaultConfig)
 
   if (result.i18n) {
     const { i18n } = result
@@ -855,6 +866,6 @@ export default async function loadConfig(
   // reactRoot can be updated correctly even with no next.config.js
   const completeConfig = assignDefaults(defaultConfig) as NextConfigComplete
   completeConfig.configFileName = configFileName
-  setHttpAgentOptions(completeConfig.httpAgentOptions)
+  setHttpClientAndAgentOptions(completeConfig)
   return completeConfig
 }

packages/next/server/dev/next-dev-server.ts

@@ -1286,6 +1286,7 @@ export default class DevServer extends Server {
         publicRuntimeConfig,
         serverRuntimeConfig,
         httpAgentOptions,
+        experimental: { enableUndici },
       } = this.nextConfig
       const { locales, defaultLocale } = this.nextConfig.i18n || {}
 
@@ -1299,6 +1300,7 @@ export default class DevServer extends Server {
           serverRuntimeConfig,
         },
         httpAgentOptions,
+        enableUndici,
         locales,
         defaultLocale,
         originalAppPath,

packages/next/server/dev/static-paths-worker.ts

@@ -7,7 +7,7 @@ import {
   collectGenerateParams,
 } from '../../build/utils'
 import { loadComponents } from '../load-components'
-import { setHttpAgentOptions } from '../config'
+import { setHttpClientAndAgentOptions } from '../config'
 
 type RuntimeConfig = any
 
@@ -22,6 +22,7 @@ export async function loadStaticPaths({
   serverless,
   config,
   httpAgentOptions,
+  enableUndici,
   locales,
   defaultLocale,
   isAppPath,
@@ -32,6 +33,7 @@ export async function loadStaticPaths({
   serverless: boolean
   config: RuntimeConfig
   httpAgentOptions: NextConfigComplete['httpAgentOptions']
+  enableUndici: NextConfigComplete['enableUndici']
   locales?: string[]
   defaultLocale?: string
   isAppPath?: boolean
@@ -49,7 +51,10 @@ export async function loadStaticPaths({
 
   // update work memory runtime-config
   require('../../shared/lib/runtime-config').setConfig(config)
-  setHttpAgentOptions(httpAgentOptions)
+  setHttpClientAndAgentOptions({
+    httpAgentOptions,
+    experimental: { enableUndici },
+  })
 
   const components = await loadComponents({
     distDir,

packages/next/server/node-polyfill-fetch.js

@@ -1,23 +1,56 @@
-import fetch, {
-  Headers,
-  Request,
-  Response,
-} from 'next/dist/compiled/node-fetch'
-
 // Polyfill fetch() in the Node.js environment
+
 if (!global.fetch) {
-  const agent = ({ protocol }) =>
-    protocol === 'http:' ? global.__NEXT_HTTP_AGENT : global.__NEXT_HTTPS_AGENT
-  const fetchWithAgent = (url, opts, ...rest) => {
-    if (!opts) {
-      opts = { agent }
-    } else if (!opts.agent) {
-      opts.agent = agent
+  function getFetchImpl() {
+    return global.__NEXT_USE_UNDICI
+      ? require('next/dist/compiled/undici')
+      : require('next/dist/compiled/node-fetch')
+  }
+  // Due to limitation of global configuration, we have to do this resolution at runtime
+  global.fetch = (...args) => {
+    const fetchImpl = getFetchImpl()
+
+    if (global.__NEXT_USE_UNDICI) {
+      // Undici does not support the `keepAlive` option,
+      // instead we have to pass a custom dispatcher
+      if (
+        !global.__NEXT_HTTP_AGENT_OPTIONS?.keepAlive &&
+        !global.__NEXT_UNDICI_AGENT_SET
+      ) {
+        global.__NEXT_UNDICI_AGENT_SET = true
+        fetchImpl.setGlobalDispatcher(new fetchImpl.Agent({ pipelining: 0 }))
+      }
+      return fetchImpl.fetch(...args)
     }
-    return fetch(url, opts, ...rest)
+    const agent = ({ protocol }) =>
+      protocol === 'http:'
+        ? global.__NEXT_HTTP_AGENT
+        : global.__NEXT_HTTPS_AGENT
+
+    if (!args[1]) {
+      args[1] = { agent }
+    } else if (!args[1].agent) {
+      args[1].agent = agent
+    }
+
+    return fetchImpl(...args)
   }
-  global.fetch = fetchWithAgent
-  global.Headers = Headers
-  global.Request = Request
-  global.Response = Response
+
+  Object.defineProperties(global, {
+    Headers: {
+      get() {
+        return getFetchImpl().Headers
+      },
+    },
+    Request: {
+      get() {
+        return getFetchImpl().Request
+      },
+    },
+    Response: {
+      get() {
+        return getFetchImpl().Response
+      },
+    },
+  })
 }