feat(optimizer): support glob includes (#12414)

Author: bluwy

docs/config/dep-optimization-options.md

@@ -35,6 +35,16 @@ export default defineConfig({
 
 By default, linked packages not inside `node_modules` are not pre-bundled. Use this option to force a linked package to be pre-bundled.
 
+**Experimental:** If you're using a library with many deep imports, you can also specify a trailing glob pattern to pre-bundle all deep imports at once. This will avoid constantly pre-bundling whenever a new deep import is used. For example:
+
+```js
+export default defineConfig({
+  optimizeDeps: {
+    include: ['my-lib/components/**/*.vue'],
+  },
+})
+```
+
 ## optimizeDeps.esbuildOptions
 
 - **Type:** [`EsbuildBuildOptions`](https://esbuild.github.io/api/#simple-options)

packages/vite/src/node/optimizer/index.ts

@@ -7,6 +7,7 @@ import colors from 'picocolors'
 import type { BuildContext, BuildOptions as EsbuildBuildOptions } from 'esbuild'
 import esbuild, { build } from 'esbuild'
 import { init, parse } from 'es-module-lexer'
+import glob from 'fast-glob'
 import { createFilter } from '@rollup/pluginutils'
 import { getDepOptimizationConfig } from '../config'
 import type { ResolvedConfig } from '../config'
@@ -25,9 +26,9 @@ import {
 } from '../utils'
 import { transformWithEsbuild } from '../plugins/esbuild'
 import { ESBUILD_MODULES_TARGET } from '../constants'
-import { resolvePackageData } from '../packages'
 import { esbuildCjsExternalPlugin, esbuildDepPlugin } from './esbuildDepPlugin'
 import { scanImports } from './scan'
+import { createOptimizeDepsIncludeResolver, expandGlobIds } from './resolve'
 export {
   initDepsOptimizer,
   initDevSsrDepsOptimizer,
@@ -844,8 +845,19 @@ export async function addManuallyIncludedOptimizeDeps(
         )
       }
     }
+
+    const includes = [...optimizeDepsInclude, ...extra]
+    for (let i = 0; i < includes.length; i++) {
+      const id = includes[i]
+      if (glob.isDynamicPattern(id)) {
+        const globIds = expandGlobIds(id, config)
+        includes.splice(i, 1, ...globIds)
+        i += globIds.length - 1
+      }
+    }
+
     const resolve = createOptimizeDepsIncludeResolver(config, ssr)
-    for (const id of [...optimizeDepsInclude, ...extra]) {
+    for (const id of includes) {
       // normalize 'foo   >bar` as 'foo > bar' to prevent same id being added
       // and for pretty printing
       const normalizedId = normalizeId(id)
@@ -867,55 +879,6 @@ export async function addManuallyIncludedOptimizeDeps(
   }
 }
 
-function createOptimizeDepsIncludeResolver(
-  config: ResolvedConfig,
-  ssr: boolean,
-) {
-  const resolve = config.createResolver({
-    asSrc: false,
-    scan: true,
-    ssrOptimizeCheck: ssr,
-    ssrConfig: config.ssr,
-    packageCache: new Map(),
-  })
-  return async (id: string) => {
-    const lastArrowIndex = id.lastIndexOf('>')
-    if (lastArrowIndex === -1) {
-      return await resolve(id, undefined, undefined, ssr)
-    }
-    // split nested selected id by last '>', for example:
-    // 'foo > bar > baz' => 'foo > bar' & 'baz'
-    const nestedRoot = id.substring(0, lastArrowIndex).trim()
-    const nestedPath = id.substring(lastArrowIndex + 1).trim()
-    const basedir = nestedResolveBasedir(
-      nestedRoot,
-      config.root,
-      config.resolve.preserveSymlinks,
-    )
-    return await resolve(
-      nestedPath,
-      path.resolve(basedir, 'package.json'),
-      undefined,
-      ssr,
-    )
-  }
-}
-
-/**
- * Continously resolve the basedir of packages separated by '>'
- */
-function nestedResolveBasedir(
-  id: string,
-  basedir: string,
-  preserveSymlinks = false,
-) {
-  const pkgs = id.split('>').map((pkg) => pkg.trim())
-  for (const pkg of pkgs) {
-    basedir = resolvePackageData(pkg, basedir, preserveSymlinks)?.dir || basedir
-  }
-  return basedir
-}
-
 export function newDepOptimizationProcessing(): DepOptimizationProcessing {
   let resolve: () => void
   const promise = new Promise((_resolve) => {

packages/vite/src/node/optimizer/resolve.ts

@@ -0,0 +1,173 @@
+import path from 'node:path'
+import glob from 'fast-glob'
+import micromatch from 'micromatch'
+import type { ResolvedConfig } from '../config'
+import { escapeRegex, getNpmPackageName, slash } from '../utils'
+import { resolvePackageData } from '../packages'
+
+export function createOptimizeDepsIncludeResolver(
+  config: ResolvedConfig,
+  ssr: boolean,
+): (id: string) => Promise<string | undefined> {
+  const resolve = config.createResolver({
+    asSrc: false,
+    scan: true,
+    ssrOptimizeCheck: ssr,
+    ssrConfig: config.ssr,
+    packageCache: new Map(),
+  })
+  return async (id: string) => {
+    const lastArrowIndex = id.lastIndexOf('>')
+    if (lastArrowIndex === -1) {
+      return await resolve(id, undefined, undefined, ssr)
+    }
+    // split nested selected id by last '>', for example:
+    // 'foo > bar > baz' => 'foo > bar' & 'baz'
+    const nestedRoot = id.substring(0, lastArrowIndex).trim()
+    const nestedPath = id.substring(lastArrowIndex + 1).trim()
+    const basedir = nestedResolveBasedir(
+      nestedRoot,
+      config.root,
+      config.resolve.preserveSymlinks,
+    )
+    return await resolve(
+      nestedPath,
+      path.resolve(basedir, 'package.json'),
+      undefined,
+      ssr,
+    )
+  }
+}
+
+/**
+ * Expand the glob syntax in `optimizeDeps.include` to proper import paths
+ */
+export function expandGlobIds(id: string, config: ResolvedConfig): string[] {
+  const pkgName = getNpmPackageName(id)
+  if (!pkgName) return []
+
+  const pkgData = resolvePackageData(
+    pkgName,
+    config.root,
+    config.resolve.preserveSymlinks,
+    config.packageCache,
+  )
+  if (!pkgData) return []
+
+  const pattern = '.' + id.slice(pkgName.length)
+  const exports = pkgData.data.exports
+
+  // if package has exports field, get all possible export paths and apply
+  // glob on them with micromatch
+  if (exports) {
+    if (typeof exports === 'string' || Array.isArray(exports)) {
+      return [pkgName]
+    }
+
+    const possibleExportPaths: string[] = []
+    for (const key in exports) {
+      if (key.startsWith('.')) {
+        if (key.includes('*')) {
+          // "./glob/*": {
+          //   "browser": "./dist/glob/*-browser/*.js", <-- get this one
+          //   "default": "./dist/glob/*/*.js"
+          // }
+          // NOTE: theoretically the "default" condition could map to a different
+          // set of files, but that complicates the resolve logic, so we assume
+          // all conditions map to the same set of files, and get the first one.
+          const exportsValue = getFirstExportStringValue(exports[key])
+          if (!exportsValue) continue
+
+          // "./dist/glob/*-browser/*.js" => "./dist/glob/**/*-browser/**/*.js"
+          // NOTE: in some cases, this could expand to consecutive /**/*/**/* etc
+          // but it's fine since fast-glob handles it the same.
+          const exportValuePattern = exportsValue.replace(/\*/g, '**/*')
+          // "./dist/glob/*-browser/*.js" => /dist\/glob\/(.*)-browser\/(.*)\.js/
+          const exportsValueGlobRe = new RegExp(
+            exportsValue.split('*').map(escapeRegex).join('(.*)'),
+          )
+
+          possibleExportPaths.push(
+            ...glob
+              .sync(exportValuePattern, {
+                cwd: pkgData.dir,
+                ignore: ['node_modules'],
+              })
+              .map((filePath) => {
+                // "./glob/*": "./dist/glob/*-browser/*.js"
+                // `filePath`: "./dist/glob/foo-browser/foo.js"
+                // we need to revert the file path back to the export key by
+                // matching value regex and replacing the capture groups to the key
+                const matched = slash(filePath).match(exportsValueGlobRe)
+                // `matched`: [..., 'foo', 'foo']
+                if (matched) {
+                  let allGlobSame = matched.length === 2
+                  // exports key can only have one *, so for >=2 matched groups,
+                  // make sure they have the same value
+                  if (!allGlobSame) {
+                    // assume true, if one group is different, set false and break
+                    allGlobSame = true
+                    for (let i = 2; i < matched.length; i++) {
+                      if (matched[i] !== matched[i - 1]) {
+                        allGlobSame = false
+                        break
+                      }
+                    }
+                  }
+                  if (allGlobSame) {
+                    return key.replace('*', matched[1]).slice(2)
+                  }
+                }
+                return ''
+              })
+              .filter(Boolean),
+          )
+        } else {
+          possibleExportPaths.push(key.slice(2))
+        }
+      }
+    }
+
+    const matched = micromatch(possibleExportPaths, pattern).map((match) =>
+      path.posix.join(pkgName, match),
+    )
+    matched.unshift(pkgName)
+    return matched
+  } else {
+    // for packages without exports, we can do a simple glob
+    const matched = glob
+      .sync(pattern, { cwd: pkgData.dir, ignore: ['node_modules'] })
+      .map((match) => path.posix.join(pkgName, slash(match)))
+    matched.unshift(pkgName)
+    return matched
+  }
+}
+
+function getFirstExportStringValue(
+  obj: string | string[] | Record<string, any>,
+): string | undefined {
+  if (typeof obj === 'string') {
+    return obj
+  } else if (Array.isArray(obj)) {
+    return obj[0]
+  } else {
+    for (const key in obj) {
+      return getFirstExportStringValue(obj[key])
+    }
+  }
+}
+
+/**
+ * Continuously resolve the basedir of packages separated by '>'
+ */
+function nestedResolveBasedir(
+  id: string,
+  basedir: string,
+  preserveSymlinks = false,
+) {
+  const pkgs = id.split('>').map((pkg) => pkg.trim())
+  for (const pkg of pkgs) {
+    basedir = resolvePackageData(pkg, basedir, preserveSymlinks)?.dir || basedir
+  }
+  return basedir
+}

packages/vite/src/node/ssr/ssrExternal.ts

@@ -7,6 +7,7 @@ import {
   bareImportRE,
   createDebugger,
   createFilter,
+  getNpmPackageName,
   isBuiltin,
   isDefined,
   isInNodeModules,
@@ -341,13 +342,3 @@ export function cjsShouldExternalizeForSSR(
   })
   return should
 }
-
-function getNpmPackageName(importPath: string): string | null {
-  const parts = importPath.split('/')
-  if (parts[0][0] === '@') {
-    if (!parts[1]) return null
-    return `${parts[0]}/${parts[1]}`
-  } else {
-    return parts[0]
-  }
-}

packages/vite/src/node/utils.ts

@@ -1238,6 +1238,16 @@ export function evalValue<T = any>(rawValue: string): T {
   return fn()
 }
 
+export function getNpmPackageName(importPath: string): string | null {
+  const parts = importPath.split('/')
+  if (parts[0][0] === '@') {
+    if (!parts[1]) return null
+    return `${parts[0]}/${parts[1]}`
+  } else {
+    return parts[0]
+  }
+}
+
 const escapeRegexRE = /[-/\\^$*+?.()|[\]{}]/g
 export function escapeRegex(str: string): string {
   return str.replace(escapeRegexRE, '\\$&')

playground/optimize-deps/__tests__/optimize-deps.spec.ts

@@ -1,11 +1,12 @@
-import { expect, test } from 'vitest'
+import { describe, expect, test } from 'vitest'
 import {
   browserErrors,
   browserLogs,
   getColor,
   isBuild,
   isServe,
   page,
+  readDepOptimizationMetadata,
   serverLogs,
   viteTestUrl,
 } from '~utils'
@@ -222,3 +223,22 @@ test.runIf(isBuild)('no missing deps during build', async () => {
     expect(log).not.toMatch('Missing dependency found after crawling ended')
   })
 })
+
+describe.runIf(isServe)('optimizeDeps config', () => {
+  test('supports include glob syntax', () => {
+    const metadata = readDepOptimizationMetadata()
+    expect(Object.keys(metadata.optimized)).to.include.members([
+      '@vitejs/test-dep-optimize-exports-with-glob',
+      '@vitejs/test-dep-optimize-exports-with-glob/named',
+      '@vitejs/test-dep-optimize-exports-with-glob/glob-dir/foo',
+      '@vitejs/test-dep-optimize-exports-with-glob/glob-dir/bar',
+      '@vitejs/test-dep-optimize-exports-with-glob/glob-dir/nested/baz',
+      '@vitejs/test-dep-optimize-with-glob',
+      '@vitejs/test-dep-optimize-with-glob/index.js',
+      '@vitejs/test-dep-optimize-with-glob/named.js',
+      '@vitejs/test-dep-optimize-with-glob/glob/foo.js',
+      '@vitejs/test-dep-optimize-with-glob/glob/bar.js',
+      '@vitejs/test-dep-optimize-with-glob/glob/nested/baz.js',
+    ])
+  })
+})

playground/optimize-deps/dep-optimize-exports-with-glob/glob/bar.js

@@ -0,0 +1,11 @@
+{
+  "name": "@vitejs/test-dep-optimize-exports-with-glob",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "exports": {
+    ".": "./index.js",
+    "./named": "./named.js",
+    "./glob-dir/*": "./glob/*.js"
+  }
+}

playground/optimize-deps/dep-optimize-exports-with-glob/glob/foo.js

@@ -0,0 +1,6 @@
+{
+  "name": "@vitejs/test-dep-optimize-with-glob",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module"
+}

playground/optimize-deps/dep-optimize-exports-with-glob/glob/nested/baz.js

@@ -23,6 +23,8 @@
     "@vitejs/test-dep-linked-include": "link:./dep-linked-include",
     "@vitejs/test-dep-node-env": "file:./dep-node-env",
     "@vitejs/test-dep-not-js": "file:./dep-not-js",
+    "@vitejs/test-dep-optimize-exports-with-glob": "file:./dep-optimize-exports-with-glob",
+    "@vitejs/test-dep-optimize-with-glob": "file:./dep-optimize-with-glob",
     "@vitejs/test-dep-relative-to-main": "file:./dep-relative-to-main",
     "@vitejs/test-dep-with-builtin-module-cjs": "file:./dep-with-builtin-module-cjs",
     "@vitejs/test-dep-with-builtin-module-esm": "file:./dep-with-builtin-module-esm",

playground/optimize-deps/dep-optimize-exports-with-glob/index.js

@@ -23,6 +23,8 @@ export default defineConfig({
       '@vitejs/test-nested-exclude > @vitejs/test-nested-include',
       // will throw if optimized (should log warning instead)
       '@vitejs/test-non-optimizable-include',
+      '@vitejs/test-dep-optimize-exports-with-glob/**/*',
+      '@vitejs/test-dep-optimize-with-glob/**/*.js',
     ],
     exclude: ['@vitejs/test-nested-exclude', '@vitejs/test-dep-non-optimized'],
     esbuildOptions: {

playground/optimize-deps/dep-optimize-exports-with-glob/named.js

@@ -5,7 +5,7 @@ import fs from 'node:fs'
 import path from 'node:path'
 import colors from 'css-color-names'
 import type { ConsoleMessage, ElementHandle } from 'playwright-chromium'
-import type { Manifest } from 'vite'
+import type { DepOptimizationMetadata, Manifest } from 'vite'
 import { normalizePath } from 'vite'
 import { fromComment } from 'convert-source-map'
 import { expect } from 'vitest'
@@ -153,6 +153,15 @@ export function readManifest(base = ''): Manifest {
   )
 }
 
+export function readDepOptimizationMetadata(): DepOptimizationMetadata {
+  return JSON.parse(
+    fs.readFileSync(
+      path.join(testDir, 'node_modules/.vite/deps/_metadata.json'),
+      'utf-8',
+    ),
+  )
+}
+
 /**
  * Poll a getter until the value it returns includes the expected value.
  */