perf: improve package cache usage (#12512)

packages/vite/src/node/optimizer/esbuildDepPlugin.ts

@@ -2,7 +2,7 @@ import path from 'node:path'
 import type { ImportKind, Plugin } from 'esbuild'
 import { CSS_LANGS_RE, KNOWN_ASSET_TYPES } from '../constants'
 import { getDepOptimizationConfig } from '..'
-import type { ResolvedConfig } from '..'
+import type { PackageCache, ResolvedConfig } from '..'
 import {
   flattenId,
   isBuiltin,
@@ -57,14 +57,24 @@ export function esbuildDepPlugin(
     ? externalTypes.filter((type) => !extensions?.includes('.' + type))
     : externalTypes
 
+  // use separate package cache for optimizer as it caches paths around node_modules
+  // and it's unlikely for the core Vite process to traverse into node_modules again
+  const esmPackageCache: PackageCache = new Map()
+  const cjsPackageCache: PackageCache = new Map()
+
   // default resolver which prefers ESM
-  const _resolve = config.createResolver({ asSrc: false, scan: true })
+  const _resolve = config.createResolver({
+    asSrc: false,
+    scan: true,
+    packageCache: esmPackageCache,
+  })
 
   // cjs resolver that prefers Node
   const _resolveRequire = config.createResolver({
     asSrc: false,
     isRequire: true,
     scan: true,
+    packageCache: cjsPackageCache,
   })
 
   const resolve = (
@@ -116,6 +126,12 @@ export function esbuildDepPlugin(
   return {
     name: 'vite:dep-pre-bundle',
     setup(build) {
+      // clear package cache when esbuild is finished
+      build.onEnd(() => {
+        esmPackageCache.clear()
+        cjsPackageCache.clear()
+      })
+
       // externalize assets and commonly known non-js file types
       // See #8459 for more details about this require-import conversion
       build.onResolve(

packages/vite/src/node/optimizer/index.ts

@@ -26,7 +26,7 @@ import {
 } from '../utils'
 import { transformWithEsbuild } from '../plugins/esbuild'
 import { ESBUILD_MODULES_TARGET } from '../constants'
-import { resolvePkgJsonPath } from '../packages'
+import { resolvePackageData } from '../packages'
 import { esbuildCjsExternalPlugin, esbuildDepPlugin } from './esbuildDepPlugin'
 import { scanImports } from './scan'
 export {
@@ -855,7 +855,7 @@ function createOptimizeDepsIncludeResolver(
     // 'foo > bar > baz' => 'foo > bar' & 'baz'
     const nestedRoot = id.substring(0, lastArrowIndex).trim()
     const nestedPath = id.substring(lastArrowIndex + 1).trim()
-    const basedir = nestedResolvePkgJsonPath(
+    const basedir = nestedResolveBasedir(
       nestedRoot,
       config.root,
       config.resolve.preserveSymlinks,
@@ -865,16 +865,16 @@ function createOptimizeDepsIncludeResolver(
 }
 
 /**
- * Like `resolvePkgJsonPath`, but supports resolving nested package names with '>'
+ * Continously resolve the basedir of packages separated by '>'
  */
-function nestedResolvePkgJsonPath(
+function nestedResolveBasedir(
   id: string,
   basedir: string,
   preserveSymlinks = false,
 ) {
   const pkgs = id.split('>').map((pkg) => pkg.trim())
   for (const pkg of pkgs) {
-    basedir = resolvePkgJsonPath(pkg, basedir, preserveSymlinks) || basedir
+    basedir = resolvePackageData(pkg, basedir, preserveSymlinks)?.dir || basedir
   }
   return basedir
 }

packages/vite/src/node/packages.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs'
 import path from 'node:path'
 import { createRequire } from 'node:module'
-import { createDebugger, createFilter, safeRealpathSync } from './utils'
+import { createFilter, safeRealpathSync } from './utils'
 import type { ResolvedConfig } from './config'
 import type { Plugin } from './plugin'
 
@@ -13,11 +13,6 @@ if (process.versions.pnp) {
   } catch {}
 }
 
-const isDebug = process.env.DEBUG
-const debug = createDebugger('vite:resolve-details', {
-  onlyWhenFocused: true,
-})
-
 /** Cache for package.json resolution and package.json contents */
 export type PackageCache = Map<string, PackageData>
 
@@ -56,49 +51,99 @@ export function invalidatePackageData(
 }
 
 export function resolvePackageData(
-  id: string,
+  pkgName: string,
   basedir: string,
   preserveSymlinks = false,
   packageCache?: PackageCache,
 ): PackageData | null {
-  let pkg: PackageData | undefined
-  let cacheKey: string | undefined
-  if (packageCache) {
-    cacheKey = `${id}&${basedir}&${preserveSymlinks}`
-    if ((pkg = packageCache.get(cacheKey))) {
-      return pkg
-    }
+  if (pnp) {
+    const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks)
+    if (packageCache?.has(cacheKey)) return packageCache.get(cacheKey)!
+
+    const pkg = pnp.resolveToUnqualified(pkgName, basedir)
+    if (!pkg) return null
+
+    const pkgData = loadPackageData(path.join(pkg, 'package.json'))
+    packageCache?.set(cacheKey, pkgData)
+
+    return pkgData
   }
-  const pkgPath = resolvePkgJsonPath(id, basedir, preserveSymlinks)
-  if (!pkgPath) return null
-  try {
-    pkg = loadPackageData(pkgPath, true, packageCache)
+
+  const originalBasedir = basedir
+  while (basedir) {
     if (packageCache) {
-      packageCache.set(cacheKey!, pkg)
-    }
-    return pkg
-  } catch (e) {
-    if (e instanceof SyntaxError) {
-      isDebug && debug(`Parsing failed: ${pkgPath}`)
+      const cached = getRpdCache(
+        packageCache,
+        pkgName,
+        basedir,
+        originalBasedir,
+        preserveSymlinks,
+      )
+      if (cached) return cached
     }
-    throw e
+
+    const pkg = path.join(basedir, 'node_modules', pkgName, 'package.json')
+    try {
+      if (fs.existsSync(pkg)) {
+        const pkgPath = preserveSymlinks ? pkg : safeRealpathSync(pkg)
+        const pkgData = loadPackageData(pkgPath)
+
+        if (packageCache) {
+          setRpdCache(
+            packageCache,
+            pkgData,
+            pkgName,
+            basedir,
+            originalBasedir,
+            preserveSymlinks,
+          )
+        }
+
+        return pkgData
+      }
+    } catch {}
+
+    const nextBasedir = path.dirname(basedir)
+    if (nextBasedir === basedir) break
+    basedir = nextBasedir
   }
+
+  return null
 }
 
-export function loadPackageData(
-  pkgPath: string,
-  preserveSymlinks?: boolean,
+export function findNearestPackageData(
+  basedir: string,
   packageCache?: PackageCache,
-): PackageData {
-  if (!preserveSymlinks) {
-    pkgPath = safeRealpathSync(pkgPath)
-  }
+): PackageData | null {
+  const originalBasedir = basedir
+  while (basedir) {
+    if (packageCache) {
+      const cached = getFnpdCache(packageCache, basedir, originalBasedir)
+      if (cached) return cached
+    }
+
+    const pkgPath = path.join(basedir, 'package.json')
+    try {
+      if (fs.statSync(pkgPath, { throwIfNoEntry: false })?.isFile()) {
+        const pkgData = loadPackageData(pkgPath)
+
+        if (packageCache) {
+          setFnpdCache(packageCache, pkgData, basedir, originalBasedir)
+        }
 
-  let cached: PackageData | undefined
-  if ((cached = packageCache?.get(pkgPath))) {
-    return cached
+        return pkgData
+      }
+    } catch {}
+
+    const nextBasedir = path.dirname(basedir)
+    if (nextBasedir === basedir) break
+    basedir = nextBasedir
   }
 
+  return null
+}
+
+export function loadPackageData(pkgPath: string): PackageData {
   const data = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'))
   const pkgDir = path.dirname(pkgPath)
   const { sideEffects } = data
@@ -147,7 +192,6 @@ export function loadPackageData(
     },
   }
 
-  packageCache?.set(pkgPath, pkg)
   return pkg
 }
 
@@ -184,29 +228,104 @@ export function watchPackageDataPlugin(config: ResolvedConfig): Plugin {
   }
 }
 
-export function resolvePkgJsonPath(
+/**
+ * Get cached `resolvePackageData` value based on `basedir`. When one is found,
+ * and we've already traversed some directories between `basedir` and `originalBasedir`,
+ * we cache the value for those in-between directories as well.
+ *
+ * This makes it so the fs is only read once for a shared `basedir`.
+ */
+function getRpdCache(
+  packageCache: PackageCache,
   pkgName: string,
   basedir: string,
-  preserveSymlinks = false,
-): string | undefined {
-  if (pnp) {
-    const pkg = pnp.resolveToUnqualified(pkgName, basedir)
-    if (!pkg) return undefined
-    return path.join(pkg, 'package.json')
+  originalBasedir: string,
+  preserveSymlinks: boolean,
+) {
+  const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks)
+  const pkgData = packageCache.get(cacheKey)
+  if (pkgData) {
+    traverseBetweenDirs(originalBasedir, basedir, (dir) => {
+      packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData)
+    })
+    return pkgData
   }
+}
 
-  let root = basedir
-  while (root) {
-    const pkg = path.join(root, 'node_modules', pkgName, 'package.json')
-    try {
-      if (fs.existsSync(pkg)) {
-        return preserveSymlinks ? pkg : safeRealpathSync(pkg)
-      }
-    } catch {}
-    const nextRoot = path.dirname(root)
-    if (nextRoot === root) break
-    root = nextRoot
+function setRpdCache(
+  packageCache: PackageCache,
+  pkgData: PackageData,
+  pkgName: string,
+  basedir: string,
+  originalBasedir: string,
+  preserveSymlinks: boolean,
+) {
+  packageCache.set(getRpdCacheKey(pkgName, basedir, preserveSymlinks), pkgData)
+  traverseBetweenDirs(originalBasedir, basedir, (dir) => {
+    packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData)
+  })
+}
+
+// package cache key for `resolvePackageData`
+function getRpdCacheKey(
+  pkgName: string,
+  basedir: string,
+  preserveSymlinks: boolean,
+) {
+  return `rpd_${pkgName}_${basedir}_${preserveSymlinks}`
+}
+
+/**
+ * Get cached `findNearestPackageData` value based on `basedir`. When one is found,
+ * and we've already traversed some directories between `basedir` and `originalBasedir`,
+ * we cache the value for those in-between directories as well.
+ *
+ * This makes it so the fs is only read once for a shared `basedir`.
+ */
+function getFnpdCache(
+  packageCache: PackageCache,
+  basedir: string,
+  originalBasedir: string,
+) {
+  const cacheKey = getFnpdCacheKey(basedir)
+  const pkgData = packageCache.get(cacheKey)
+  if (pkgData) {
+    traverseBetweenDirs(originalBasedir, basedir, (dir) => {
+      packageCache.set(getFnpdCacheKey(dir), pkgData)
+    })
+    return pkgData
   }
+}
 
-  return undefined
+function setFnpdCache(
+  packageCache: PackageCache,
+  pkgData: PackageData,
+  basedir: string,
+  originalBasedir: string,
+) {
+  packageCache.set(getFnpdCacheKey(basedir), pkgData)
+  traverseBetweenDirs(originalBasedir, basedir, (dir) => {
+    packageCache.set(getFnpdCacheKey(dir), pkgData)
+  })
+}
+
+// package cache key for `findNearestPackageData`
+function getFnpdCacheKey(basedir: string) {
+  return `fnpd_${basedir}`
+}
+
+/**
+ * Traverse between `longerDir` (inclusive) and `shorterDir` (exclusive) and call `cb` for each dir.
+ * @param longerDir Longer dir path, e.g. `/User/foo/bar/baz`
+ * @param shorterDir Shorter dir path, e.g. `/User/foo`
+ */
+function traverseBetweenDirs(
+  longerDir: string,
+  shorterDir: string,
+  cb: (dir: string) => void,
+) {
+  while (longerDir !== shorterDir) {
+    cb(longerDir)
+    longerDir = path.dirname(longerDir)
+  }
 }