Add transpilePackages option (#41583)

This is a new experimental feature to specify a list of packages (or
subpaths in packages, like `pkg/src/index.ts`), to opt-in Next.js
transpilation in the server build.

cc @jescalan 

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

packages/next/build/webpack-config.ts

@@ -119,7 +119,9 @@ function errorIfEnvConflicted(config: NextConfigComplete, key: string) {
 
 function isResourceInPackages(resource: string, packageNames?: string[]) {
   return packageNames?.some((p: string) =>
-    new RegExp('[/\\\\]node_modules[/\\\\]' + p + '[/\\\\]').test(resource)
+    resource.includes(
+      path.sep + pathJoin('node_modules', p.replace(/\//g, path.sep)) + path.sep
+    )
   )
 }
 
@@ -1168,21 +1170,38 @@ export default async function getBaseWebpackConfig(
       return
     }
 
+    // If a package should be transpiled by Next.js, we skip making it external.
+    // It doesn't matter what the extension is, as we'll transpile it anyway.
+    const shouldBeBundled = isResourceInPackages(
+      res,
+      config.experimental.transpilePackages
+    )
+
     if (/node_modules[/\\].*\.[mc]?js$/.test(res)) {
       if (layer === WEBPACK_LAYERS.server) {
         // All packages should be bundled for the server layer if they're not opted out.
-        if (isResourceInPackages(res, optoutBundlingPackages)) {
+        // This option takes priority over the transpilePackages option.
+        if (
+          isResourceInPackages(
+            res,
+            config.experimental.serverComponentsExternalPackages
+          )
+        ) {
           return `${externalType} ${request}`
         }
 
         return
       }
 
+      if (shouldBeBundled) return
+
       // Anything else that is standard JavaScript within `node_modules`
       // can be externalized.
       return `${externalType} ${request}`
     }
 
+    if (shouldBeBundled) return
+
     // Default behavior: bundle the code!
   }
 
@@ -1196,6 +1215,13 @@ export default async function getBaseWebpackConfig(
       if (babelIncludeRegexes.some((r) => r.test(excludePath))) {
         return false
       }
+
+      const shouldBeBundled = isResourceInPackages(
+        excludePath,
+        config.experimental.transpilePackages
+      )
+      if (shouldBeBundled) return false
+
       return excludePath.includes('node_modules')
     },
   }

packages/next/server/config-schema.ts

@@ -348,6 +348,12 @@ const configSchema = {
           },
           type: 'array',
         },
+        transpilePackages: {
+          items: {
+            type: 'string',
+          },
+          type: 'array',
+        },
         scrollRestoration: {
           type: 'boolean',
         },

packages/next/server/config-shared.ts

@@ -159,6 +159,9 @@ export interface ExperimentalConfig {
   // A list of packages that should be treated as external in the RSC server build
   serverComponentsExternalPackages?: string[]
 
+  // A list of packages that should always be transpiled and bundled in the server
+  transpilePackages?: string[]
+
   fontLoaders?: [{ loader: string; options?: any }]
 
   webVitalsAttribution?: Array<typeof WEB_VITALS[number]>

test/e2e/app-dir/rsc-external.test.ts

@@ -84,6 +84,11 @@ describe('app dir - rsc external dependency', () => {
     )
   })
 
+  it('should transpile specific external packages with the `transpilePackages` option', async () => {
+    const clientHtml = await renderViaHTTP(next.url, '/external-imports/client')
+    expect(clientHtml).toContain('transpilePackages:5')
+  })
+
   it('should resolve the subset react in server components based on the react-server condition', async () => {
     await fetchViaHTTP(next.url, '/react-server').then(async (response) => {
       const result = await resolveStreamResponse(response)

test/e2e/app-dir/rsc-external/app/external-imports/client/page.js

@@ -2,6 +2,8 @@
 
 import getType, { named, value, array, obj } from 'non-isomorphic-text'
 
+import add from 'untranspiled-module'
+
 export default function Page() {
   return (
     <div>
@@ -10,6 +12,7 @@ export default function Page() {
       <div>{`export value:${value}`}</div>
       <div>{`export array:${array.join(',')}`}</div>
       <div>{`export object:{x:${obj.x}}`}</div>
+      <div>{`transpilePackages:${add(2, 3)}`}</div>
     </div>
   )
 }

test/e2e/app-dir/rsc-external/next.config.js

@@ -3,5 +3,6 @@ module.exports = {
   experimental: {
     appDir: true,
     serverComponentsExternalPackages: ['conditional-exports-optout'],
+    transpilePackages: ['untranspiled-module'],
   },
 }

test/e2e/app-dir/rsc-external/node_modules_bak/untranspiled-module/index.ts

@@ -0,0 +1,3 @@
+export default function add(a: number, b: number) {
+  return a + b
+}

test/e2e/app-dir/rsc-external/node_modules_bak/untranspiled-module/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "untranspiled-module",
+  "main": "index.ts"
+}