feat: non-blocking needs interop (#7568)

Author: patak-dev

packages/vite/src/node/index.ts

@@ -36,7 +36,8 @@ export type {
   DepOptimizationResult,
   DepOptimizationProcessing,
   OptimizedDepInfo,
-  OptimizedDeps
+  OptimizedDeps,
+  ExportsData
 } from './optimizer'
 export type { Plugin } from './plugin'
 export type { PackageCache, PackageData } from './packages'

packages/vite/src/node/optimizer/index.ts

@@ -34,6 +34,8 @@ export type ExportsData = ReturnType<typeof parse> & {
   // es-module-lexer has a facade detection but isn't always accurate for our
   // use case when the module has default export
   hasReExports?: true
+  // hint if the dep requires loading as jsx
+  jsxLoader?: true
 }
 
 export interface OptimizedDeps {
@@ -64,6 +66,12 @@ export interface DepOptimizationOptions {
    * cannot be globs).
    */
   exclude?: string[]
+  /**
+   * Force ESM interop when importing for these dependencies. Some legacy
+   * packages advertise themselves as ESM but use `require` internally
+   * @experimental
+   */
+  needsInterop?: string[]
   /**
    * Options to pass to esbuild during the dep scanning and optimization
    *
@@ -134,6 +142,11 @@ export interface OptimizedDepInfo {
    * but the bundles may not yet be saved to disk
    */
   processing?: Promise<void>
+  /**
+   * ExportData cache, discovered deps will parse the src entry to get exports
+   * data used both to define if interop is needed and when pre-bundling
+   */
+  exportsData?: Promise<ExportsData>
 }
 
 export interface DepOptimizationMetadata {
@@ -297,12 +310,13 @@ export async function discoverProjectDependencies(
   )
   const discovered: Record<string, OptimizedDepInfo> = {}
   for (const id in deps) {
-    const entry = deps[id]
+    const src = deps[id]
     discovered[id] = {
       id,
       file: getOptimizedDepPath(id, config),
-      src: entry,
-      browserHash: browserHash
+      src,
+      browserHash: browserHash,
+      exportsData: extractExportsData(src, config)
     }
   }
   return discovered
@@ -368,17 +382,24 @@ export async function runOptimizeDeps(
 
   const qualifiedIds = Object.keys(depsInfo)
 
-  if (!qualifiedIds.length) {
-    return {
-      metadata,
-      commit() {
-        // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps`
-        return commitProcessingDepsCacheSync()
-      },
-      cancel
+  const processingResult: DepOptimizationResult = {
+    metadata,
+    async commit() {
+      // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps`
+      // Processing is done, we can now replace the depsCacheDir with processingCacheDir
+      // Rewire the file paths from the temporal processing dir to the final deps cache dir
+      await removeDir(depsCacheDir)
+      await renameDir(processingCacheDir, depsCacheDir)
+    },
+    cancel() {
+      fs.rmSync(processingCacheDir, { recursive: true, force: true })
     }
   }
 
+  if (!qualifiedIds.length) {
+    return processingResult
+  }
+
   // esbuild generates nested directory output with lowest common ancestor base
   // this is unpredictable and makes it difficult to analyze entry / output
   // mapping. So what we do here is:
@@ -392,51 +413,20 @@ export async function runOptimizeDeps(
   const { plugins = [], ...esbuildOptions } =
     config.optimizeDeps?.esbuildOptions ?? {}
 
-  await init
   for (const id in depsInfo) {
-    const flatId = flattenId(id)
-    const filePath = (flatIdDeps[flatId] = depsInfo[id].src!)
-    let exportsData: ExportsData
-    if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) {
-      // For custom supported extensions, build the entry file to transform it into JS,
-      // and then parse with es-module-lexer. Note that the `bundle` option is not `true`,
-      // so only the entry file is being transformed.
-      const result = await build({
-        ...esbuildOptions,
-        plugins,
-        entryPoints: [filePath],
-        write: false,
-        format: 'esm'
-      })
-      exportsData = parse(result.outputFiles[0].text) as ExportsData
-    } else {
-      const entryContent = fs.readFileSync(filePath, 'utf-8')
-      try {
-        exportsData = parse(entryContent) as ExportsData
-      } catch {
-        const loader = esbuildOptions.loader?.[path.extname(filePath)] || 'jsx'
-        debug(
-          `Unable to parse dependency: ${id}. Trying again with a ${loader} transform.`
-        )
-        const transformed = await transformWithEsbuild(entryContent, filePath, {
-          loader
-        })
-        // Ensure that optimization won't fail by defaulting '.js' to the JSX parser.
-        // This is useful for packages such as Gatsby.
-        esbuildOptions.loader = {
-          '.js': 'jsx',
-          ...esbuildOptions.loader
-        }
-        exportsData = parse(transformed.code) as ExportsData
-      }
-      for (const { ss, se } of exportsData[0]) {
-        const exp = entryContent.slice(ss, se)
-        if (/export\s+\*\s+from/.test(exp)) {
-          exportsData.hasReExports = true
-        }
+    const src = depsInfo[id].src!
+    const exportsData = await (depsInfo[id].exportsData ??
+      extractExportsData(src, config))
+    if (exportsData.jsxLoader) {
+      // Ensure that optimization won't fail by defaulting '.js' to the JSX parser.
+      // This is useful for packages such as Gatsby.
+      esbuildOptions.loader = {
+        '.js': 'jsx',
+        ...esbuildOptions.loader
       }
     }
-
+    const flatId = flattenId(id)
+    flatIdDeps[flatId] = src
     idToExports[id] = exportsData
     flatIdToExports[flatId] = exportsData
   }
@@ -483,15 +473,18 @@ export async function runOptimizeDeps(
   for (const id in depsInfo) {
     const output = esbuildOutputFromId(meta.outputs, id, processingCacheDir)
 
+    const { exportsData, ...info } = depsInfo[id]
     addOptimizedDepInfo(metadata, 'optimized', {
-      ...depsInfo[id],
-      needsInterop: needsInterop(id, idToExports[id], output),
+      ...info,
       // We only need to hash the output.imports in to check for stability, but adding the hash
       // and file path gives us a unique hash that may be useful for other things in the future
       fileHash: getHash(
         metadata.hash + depsInfo[id].file + JSON.stringify(output.imports)
       ),
-      browserHash: metadata.browserHash
+      browserHash: metadata.browserHash,
+      // After bundling we have more information and can warn the user about legacy packages
+      // that require manual configuration
+      needsInterop: needsInterop(config, id, idToExports[id], output)
     })
   }
 
@@ -522,25 +515,7 @@ export async function runOptimizeDeps(
 
   debug(`deps bundled in ${(performance.now() - start).toFixed(2)}ms`)
 
-  return {
-    metadata,
-    commit() {
-      // Write metadata file, delete `deps` folder and rename the new `processing` folder to `deps` in sync
-      return commitProcessingDepsCacheSync()
-    },
-    cancel
-  }
-
-  async function commitProcessingDepsCacheSync() {
-    // Processing is done, we can now replace the depsCacheDir with processingCacheDir
-    // Rewire the file paths from the temporal processing dir to the final deps cache dir
-    await removeDir(depsCacheDir)
-    await renameDir(processingCacheDir, depsCacheDir)
-  }
-
-  function cancel() {
-    fs.rmSync(processingCacheDir, { recursive: true, force: true })
-  }
+  return processingResult
 }
 
 export async function findKnownImports(
@@ -735,17 +710,71 @@ function esbuildOutputFromId(
   ]
 }
 
+export async function extractExportsData(
+  filePath: string,
+  config: ResolvedConfig
+): Promise<ExportsData> {
+  await init
+  let exportsData: ExportsData
+
+  const esbuildOptions = config.optimizeDeps?.esbuildOptions ?? {}
+  if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) {
+    // For custom supported extensions, build the entry file to transform it into JS,
+    // and then parse with es-module-lexer. Note that the `bundle` option is not `true`,
+    // so only the entry file is being transformed.
+    const result = await build({
+      ...esbuildOptions,
+      entryPoints: [filePath],
+      write: false,
+      format: 'esm'
+    })
+    exportsData = parse(result.outputFiles[0].text) as ExportsData
+  } else {
+    const entryContent = fs.readFileSync(filePath, 'utf-8')
+    try {
+      exportsData = parse(entryContent) as ExportsData
+    } catch {
+      const loader = esbuildOptions.loader?.[path.extname(filePath)] || 'jsx'
+      debug(
+        `Unable to parse: ${filePath}.\n Trying again with a ${loader} transform.`
+      )
+      const transformed = await transformWithEsbuild(entryContent, filePath, {
+        loader
+      })
+      // Ensure that optimization won't fail by defaulting '.js' to the JSX parser.
+      // This is useful for packages such as Gatsby.
+      esbuildOptions.loader = {
+        '.js': 'jsx',
+        ...esbuildOptions.loader
+      }
+      exportsData = parse(transformed.code) as ExportsData
+      exportsData.jsxLoader = true
+    }
+    for (const { ss, se } of exportsData[0]) {
+      const exp = entryContent.slice(ss, se)
+      if (/export\s+\*\s+from/.test(exp)) {
+        exportsData.hasReExports = true
+      }
+    }
+  }
+  return exportsData
+}
+
 // https://github.com/vitejs/vite/issues/1724#issuecomment-767619642
 // a list of modules that pretends to be ESM but still uses `require`.
 // this causes esbuild to wrap them as CJS even when its entry appears to be ESM.
 const KNOWN_INTEROP_IDS = new Set(['moment'])
 
 function needsInterop(
+  config: ResolvedConfig,
   id: string,
   exportsData: ExportsData,
-  output: { exports: string[] }
+  output?: { exports: string[] }
 ): boolean {
-  if (KNOWN_INTEROP_IDS.has(id)) {
+  if (
+    config.optimizeDeps?.needsInterop?.includes(id) ||
+    KNOWN_INTEROP_IDS.has(id)
+  ) {
     return true
   }
   const [imports, exports] = exportsData
@@ -754,16 +783,19 @@ function needsInterop(
     return true
   }
 
-  // if a peer dependency used require() on a ESM dependency, esbuild turns the
-  // ESM dependency's entry chunk into a single default export... detect
-  // such cases by checking exports mismatch, and force interop.
-  const generatedExports: string[] = output.exports
-
-  if (
-    !generatedExports ||
-    (isSingleDefaultExport(generatedExports) && !isSingleDefaultExport(exports))
-  ) {
-    return true
+  if (output) {
+    // if a peer dependency used require() on a ESM dependency, esbuild turns the
+    // ESM dependency's entry chunk into a single default export... detect
+    // such cases by checking exports mismatch, and force interop.
+    const generatedExports: string[] = output.exports
+
+    if (
+      !generatedExports ||
+      (isSingleDefaultExport(generatedExports) &&
+        !isSingleDefaultExport(exports))
+    ) {
+      return true
+    }
   }
   return false
 }
@@ -846,14 +878,17 @@ function findOptimizedDepInfoInRecord(
 
 export async function optimizedDepNeedsInterop(
   metadata: DepOptimizationMetadata,
-  file: string
+  file: string,
+  config: ResolvedConfig
 ): Promise<boolean | undefined> {
   const depInfo = optimizedDepInfoFromFile(metadata, file)
-
-  if (!depInfo) return undefined
-
-  // Wait until the dependency has been pre-bundled
-  await depInfo.processing
-
+  if (depInfo?.src && depInfo.needsInterop === undefined) {
+    depInfo.exportsData ??= extractExportsData(depInfo.src, config)
+    depInfo.needsInterop = needsInterop(
+      config,
+      depInfo.id,
+      await depInfo.exportsData
+    )
+  }
   return depInfo?.needsInterop
 }

packages/vite/src/node/optimizer/registerMissing.ts

@@ -9,6 +9,7 @@ import {
   depsFromOptimizedDepInfo,
   depsLogString,
   discoverProjectDependencies,
+  extractExportsData,
   getOptimizedDepPath,
   loadCachedDepOptimizationMetadata,
   newDepOptimizationProcessing,
@@ -177,11 +178,28 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps {
 
       const newData = processingResult.metadata
 
+      const needsInteropMismatch = []
+      for (const dep in metadata.discovered) {
+        const discoveredDepInfo = metadata.discovered[dep]
+        const depInfo = newData.optimized[dep]
+        if (depInfo) {
+          if (
+            discoveredDepInfo.needsInterop !== undefined &&
+            depInfo.needsInterop !== discoveredDepInfo.needsInterop
+          ) {
+            // This only happens when a discovered dependency has mixed ESM and CJS syntax
+            // and it hasn't been manually added to optimizeDeps.needsInterop
+            needsInteropMismatch.push(dep)
+          }
+        }
+      }
+
       // After a re-optimization, if the internal bundled chunks change a full page reload
       // is required. If the files are stable, we can avoid the reload that is expensive
       // for large applications. Comparing their fileHash we can find out if it is safe to
       // keep the current browser state.
       const needsReload =
+        needsInteropMismatch.length > 0 ||
         metadata.hash !== newData.hash ||
         Object.keys(metadata.optimized).some((dep) => {
           return (
@@ -284,6 +302,19 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps {
               timestamp: true
             }
           )
+          if (needsInteropMismatch.length > 0) {
+            config.logger.warn(
+              `Mixed ESM and CJS detected in ${colors.yellow(
+                needsInteropMismatch.join(', ')
+              )}, add ${
+                needsInteropMismatch.length === 1 ? 'it' : 'them'
+              } to optimizeDeps.needsInterop to speed up cold start`,
+              {
+                timestamp: true
+              }
+            )
+          }
+
           fullReload()
         }
       }
@@ -378,7 +409,8 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps {
       ),
       // loading of this pre-bundled dep needs to await for its processing
       // promise to be resolved
-      processing: depOptimizationProcessing.promise
+      processing: depOptimizationProcessing.promise,
+      exportsData: extractExportsData(resolved, config)
     })
 
     // Debounced rerun, let other missing dependencies be discovered before

packages/vite/src/node/plugins/importAnalysis.ts

@@ -309,11 +309,6 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin {
         return [url, resolved.id]
       }
 
-      // Import rewrites, we do them after all the URLs have been resolved
-      // to help with the discovery of new dependencies. If we need to wait
-      // for each dependency there could be one reload per import
-      const importRewrites: (() => Promise<void>)[] = []
-
       for (let index = 0; index < imports.length; index++) {
         const {
           s: start,
@@ -403,75 +398,66 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin {
           server?.moduleGraph.safeModulesPath.add(fsPathFromUrl(url))
 
           if (url !== specifier) {
-            importRewrites.push(async () => {
-              let rewriteDone = false
-              if (
-                server?._optimizedDeps &&
-                isOptimizedDepFile(resolvedId, config) &&
-                !resolvedId.match(optimizedDepChunkRE)
-              ) {
-                // for optimized cjs deps, support named imports by rewriting named imports to const assignments.
-                // internal optimized chunks don't need es interop and are excluded
-
-                // The browserHash in resolvedId could be stale in which case there will be a full
-                // page reload. We could return a 404 in that case but it is safe to return the request
-                const file = cleanUrl(resolvedId) // Remove ?v={hash}
-
-                const needsInterop = await optimizedDepNeedsInterop(
-                  server._optimizedDeps!.metadata,
-                  file
-                )
-
-                if (needsInterop === undefined) {
-                  // Non-entry dynamic imports from dependencies will reach here as there isn't
-                  // optimize info for them, but they don't need es interop. If the request isn't
-                  // a dynamic import, then it is an internal Vite error
-                  if (!file.match(optimizedDepDynamicRE)) {
-                    config.logger.error(
-                      colors.red(
-                        `Vite Error, ${url} optimized info should be defined`
-                      )
-                    )
-                  }
-                } else if (needsInterop) {
-                  debug(`${url} needs interop`)
-                  if (isDynamicImport) {
-                    // rewrite `import('package')` to expose the default directly
-                    str().overwrite(
-                      expStart,
-                      expEnd,
-                      `import('${url}').then(m => m.default && m.default.__esModule ? m.default : ({ ...m.default, default: m.default }))`,
-                      { contentOnly: true }
+            let rewriteDone = false
+            if (
+              server?._optimizedDeps &&
+              isOptimizedDepFile(resolvedId, config) &&
+              !resolvedId.match(optimizedDepChunkRE)
+            ) {
+              // for optimized cjs deps, support named imports by rewriting named imports to const assignments.
+              // internal optimized chunks don't need es interop and are excluded
+
+              // The browserHash in resolvedId could be stale in which case there will be a full
+              // page reload. We could return a 404 in that case but it is safe to return the request
+              const file = cleanUrl(resolvedId) // Remove ?v={hash}
+
+              const needsInterop = await optimizedDepNeedsInterop(
+                server._optimizedDeps!.metadata,
+                file,
+                config
+              )
+
+              if (needsInterop === undefined) {
+                // Non-entry dynamic imports from dependencies will reach here as there isn't
+                // optimize info for them, but they don't need es interop. If the request isn't
+                // a dynamic import, then it is an internal Vite error
+                if (!file.match(optimizedDepDynamicRE)) {
+                  config.logger.error(
+                    colors.red(
+                      `Vite Error, ${url} optimized info should be defined`
                     )
+                  )
+                }
+              } else if (needsInterop) {
+                debug(`${url} needs interop`)
+                if (isDynamicImport) {
+                  // rewrite `import('package')` to expose the default directly
+                  str().overwrite(
+                    expStart,
+                    expEnd,
+                    `import('${url}').then(m => m.default && m.default.__esModule ? m.default : ({ ...m.default, default: m.default }))`,
+                    { contentOnly: true }
+                  )
+                } else {
+                  const exp = source.slice(expStart, expEnd)
+                  const rewritten = transformCjsImport(exp, url, rawUrl, index)
+                  if (rewritten) {
+                    str().overwrite(expStart, expEnd, rewritten, {
+                      contentOnly: true
+                    })
                   } else {
-                    const exp = source.slice(expStart, expEnd)
-                    const rewritten = transformCjsImport(
-                      exp,
-                      url,
-                      rawUrl,
-                      index
-                    )
-                    if (rewritten) {
-                      str().overwrite(expStart, expEnd, rewritten, {
-                        contentOnly: true
-                      })
-                    } else {
-                      // #1439 export * from '...'
-                      str().overwrite(start, end, url, { contentOnly: true })
-                    }
+                    // #1439 export * from '...'
+                    str().overwrite(start, end, url, { contentOnly: true })
                   }
-                  rewriteDone = true
                 }
+                rewriteDone = true
               }
-              if (!rewriteDone) {
-                str().overwrite(
-                  start,
-                  end,
-                  isDynamicImport ? `'${url}'` : url,
-                  { contentOnly: true }
-                )
-              }
-            })
+            }
+            if (!rewriteDone) {
+              str().overwrite(start, end, isDynamicImport ? `'${url}'` : url, {
+                contentOnly: true
+              })
+            }
           }
 
           // record for HMR import chain analysis
@@ -636,14 +622,6 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin {
         })
       }
 
-      // Await for import rewrites that requires dependencies to be pre-bundled to
-      // know if es interop is needed after starting further transformRequest calls
-      // This will let Vite process deeper into the user code and find more missing
-      // dependencies before the next page reload
-      for (const rewrite of importRewrites) {
-        await rewrite()
-      }
-
       if (s) {
         return {
           code: s.toString(),

playground/ssr-vue/__tests__/ssr-vue.spec.ts

@@ -117,6 +117,8 @@ test('css', async () => {
     expect(await getColor('h1')).toBe('green')
     expect(await getColor('.jsx')).toBe('blue')
   } else {
+    // During dev, the CSS is loaded from async chunk and we may have to wait
+    // when the test runs concurrently.
     await untilUpdated(() => getColor('h1'), 'green')
     await untilUpdated(() => getColor('.jsx'), 'blue')
   }