From abc2b9c2ff16937860282bab7b67fd47e0c3bb3c Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Wed, 22 Mar 2023 01:06:59 +0800 Subject: [PATCH] perf: improve package cache usage (#12512) --- .../src/node/optimizer/esbuildDepPlugin.ts | 20 +- packages/vite/src/node/optimizer/index.ts | 10 +- packages/vite/src/node/packages.ts | 229 +++++++++++++----- packages/vite/src/node/plugins/resolve.ts | 59 ++--- packages/vite/src/node/ssr/ssrExternal.ts | 27 +-- 5 files changed, 232 insertions(+), 113 deletions(-) diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index e6378bc4e94930..a01f48eb5a556b 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -2,7 +2,7 @@ import path from 'node:path' import type { ImportKind, Plugin } from 'esbuild' import { CSS_LANGS_RE, KNOWN_ASSET_TYPES } from '../constants' import { getDepOptimizationConfig } from '..' -import type { ResolvedConfig } from '..' +import type { PackageCache, ResolvedConfig } from '..' import { flattenId, isBuiltin, @@ -57,14 +57,24 @@ export function esbuildDepPlugin( ? externalTypes.filter((type) => !extensions?.includes('.' + type)) : externalTypes + // use separate package cache for optimizer as it caches paths around node_modules + // and it's unlikely for the core Vite process to traverse into node_modules again + const esmPackageCache: PackageCache = new Map() + const cjsPackageCache: PackageCache = new Map() + // default resolver which prefers ESM - const _resolve = config.createResolver({ asSrc: false, scan: true }) + const _resolve = config.createResolver({ + asSrc: false, + scan: true, + packageCache: esmPackageCache, + }) // cjs resolver that prefers Node const _resolveRequire = config.createResolver({ asSrc: false, isRequire: true, scan: true, + packageCache: cjsPackageCache, }) const resolve = ( @@ -116,6 +126,12 @@ export function esbuildDepPlugin( return { name: 'vite:dep-pre-bundle', setup(build) { + // clear package cache when esbuild is finished + build.onEnd(() => { + esmPackageCache.clear() + cjsPackageCache.clear() + }) + // externalize assets and commonly known non-js file types // See #8459 for more details about this require-import conversion build.onResolve( diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 42620fc2ff7ab2..04ba88cb38eb08 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -26,7 +26,7 @@ import { } from '../utils' import { transformWithEsbuild } from '../plugins/esbuild' import { ESBUILD_MODULES_TARGET } from '../constants' -import { resolvePkgJsonPath } from '../packages' +import { resolvePackageData } from '../packages' import { esbuildCjsExternalPlugin, esbuildDepPlugin } from './esbuildDepPlugin' import { scanImports } from './scan' export { @@ -855,7 +855,7 @@ function createOptimizeDepsIncludeResolver( // 'foo > bar > baz' => 'foo > bar' & 'baz' const nestedRoot = id.substring(0, lastArrowIndex).trim() const nestedPath = id.substring(lastArrowIndex + 1).trim() - const basedir = nestedResolvePkgJsonPath( + const basedir = nestedResolveBasedir( nestedRoot, config.root, config.resolve.preserveSymlinks, @@ -865,16 +865,16 @@ function createOptimizeDepsIncludeResolver( } /** - * Like `resolvePkgJsonPath`, but supports resolving nested package names with '>' + * Continously resolve the basedir of packages separated by '>' */ -function nestedResolvePkgJsonPath( +function nestedResolveBasedir( id: string, basedir: string, preserveSymlinks = false, ) { const pkgs = id.split('>').map((pkg) => pkg.trim()) for (const pkg of pkgs) { - basedir = resolvePkgJsonPath(pkg, basedir, preserveSymlinks) || basedir + basedir = resolvePackageData(pkg, basedir, preserveSymlinks)?.dir || basedir } return basedir } diff --git a/packages/vite/src/node/packages.ts b/packages/vite/src/node/packages.ts index a26c6e50a04350..728c941051f72f 100644 --- a/packages/vite/src/node/packages.ts +++ b/packages/vite/src/node/packages.ts @@ -1,7 +1,7 @@ import fs from 'node:fs' import path from 'node:path' import { createRequire } from 'node:module' -import { createDebugger, createFilter, safeRealpathSync } from './utils' +import { createFilter, safeRealpathSync } from './utils' import type { ResolvedConfig } from './config' import type { Plugin } from './plugin' @@ -13,11 +13,6 @@ if (process.versions.pnp) { } catch {} } -const isDebug = process.env.DEBUG -const debug = createDebugger('vite:resolve-details', { - onlyWhenFocused: true, -}) - /** Cache for package.json resolution and package.json contents */ export type PackageCache = Map @@ -56,49 +51,99 @@ export function invalidatePackageData( } export function resolvePackageData( - id: string, + pkgName: string, basedir: string, preserveSymlinks = false, packageCache?: PackageCache, ): PackageData | null { - let pkg: PackageData | undefined - let cacheKey: string | undefined - if (packageCache) { - cacheKey = `${id}&${basedir}&${preserveSymlinks}` - if ((pkg = packageCache.get(cacheKey))) { - return pkg - } + if (pnp) { + const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks) + if (packageCache?.has(cacheKey)) return packageCache.get(cacheKey)! + + const pkg = pnp.resolveToUnqualified(pkgName, basedir) + if (!pkg) return null + + const pkgData = loadPackageData(path.join(pkg, 'package.json')) + packageCache?.set(cacheKey, pkgData) + + return pkgData } - const pkgPath = resolvePkgJsonPath(id, basedir, preserveSymlinks) - if (!pkgPath) return null - try { - pkg = loadPackageData(pkgPath, true, packageCache) + + const originalBasedir = basedir + while (basedir) { if (packageCache) { - packageCache.set(cacheKey!, pkg) - } - return pkg - } catch (e) { - if (e instanceof SyntaxError) { - isDebug && debug(`Parsing failed: ${pkgPath}`) + const cached = getRpdCache( + packageCache, + pkgName, + basedir, + originalBasedir, + preserveSymlinks, + ) + if (cached) return cached } - throw e + + const pkg = path.join(basedir, 'node_modules', pkgName, 'package.json') + try { + if (fs.existsSync(pkg)) { + const pkgPath = preserveSymlinks ? pkg : safeRealpathSync(pkg) + const pkgData = loadPackageData(pkgPath) + + if (packageCache) { + setRpdCache( + packageCache, + pkgData, + pkgName, + basedir, + originalBasedir, + preserveSymlinks, + ) + } + + return pkgData + } + } catch {} + + const nextBasedir = path.dirname(basedir) + if (nextBasedir === basedir) break + basedir = nextBasedir } + + return null } -export function loadPackageData( - pkgPath: string, - preserveSymlinks?: boolean, +export function findNearestPackageData( + basedir: string, packageCache?: PackageCache, -): PackageData { - if (!preserveSymlinks) { - pkgPath = safeRealpathSync(pkgPath) - } +): PackageData | null { + const originalBasedir = basedir + while (basedir) { + if (packageCache) { + const cached = getFnpdCache(packageCache, basedir, originalBasedir) + if (cached) return cached + } + + const pkgPath = path.join(basedir, 'package.json') + try { + if (fs.statSync(pkgPath, { throwIfNoEntry: false })?.isFile()) { + const pkgData = loadPackageData(pkgPath) + + if (packageCache) { + setFnpdCache(packageCache, pkgData, basedir, originalBasedir) + } - let cached: PackageData | undefined - if ((cached = packageCache?.get(pkgPath))) { - return cached + return pkgData + } + } catch {} + + const nextBasedir = path.dirname(basedir) + if (nextBasedir === basedir) break + basedir = nextBasedir } + return null +} + +export function loadPackageData(pkgPath: string): PackageData { const data = JSON.parse(fs.readFileSync(pkgPath, 'utf-8')) const pkgDir = path.dirname(pkgPath) const { sideEffects } = data @@ -147,7 +192,6 @@ export function loadPackageData( }, } - packageCache?.set(pkgPath, pkg) return pkg } @@ -184,29 +228,104 @@ export function watchPackageDataPlugin(config: ResolvedConfig): Plugin { } } -export function resolvePkgJsonPath( +/** + * Get cached `resolvePackageData` value based on `basedir`. When one is found, + * and we've already traversed some directories between `basedir` and `originalBasedir`, + * we cache the value for those in-between directories as well. + * + * This makes it so the fs is only read once for a shared `basedir`. + */ +function getRpdCache( + packageCache: PackageCache, pkgName: string, basedir: string, - preserveSymlinks = false, -): string | undefined { - if (pnp) { - const pkg = pnp.resolveToUnqualified(pkgName, basedir) - if (!pkg) return undefined - return path.join(pkg, 'package.json') + originalBasedir: string, + preserveSymlinks: boolean, +) { + const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks) + const pkgData = packageCache.get(cacheKey) + if (pkgData) { + traverseBetweenDirs(originalBasedir, basedir, (dir) => { + packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData) + }) + return pkgData } +} - let root = basedir - while (root) { - const pkg = path.join(root, 'node_modules', pkgName, 'package.json') - try { - if (fs.existsSync(pkg)) { - return preserveSymlinks ? pkg : safeRealpathSync(pkg) - } - } catch {} - const nextRoot = path.dirname(root) - if (nextRoot === root) break - root = nextRoot +function setRpdCache( + packageCache: PackageCache, + pkgData: PackageData, + pkgName: string, + basedir: string, + originalBasedir: string, + preserveSymlinks: boolean, +) { + packageCache.set(getRpdCacheKey(pkgName, basedir, preserveSymlinks), pkgData) + traverseBetweenDirs(originalBasedir, basedir, (dir) => { + packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData) + }) +} + +// package cache key for `resolvePackageData` +function getRpdCacheKey( + pkgName: string, + basedir: string, + preserveSymlinks: boolean, +) { + return `rpd_${pkgName}_${basedir}_${preserveSymlinks}` +} + +/** + * Get cached `findNearestPackageData` value based on `basedir`. When one is found, + * and we've already traversed some directories between `basedir` and `originalBasedir`, + * we cache the value for those in-between directories as well. + * + * This makes it so the fs is only read once for a shared `basedir`. + */ +function getFnpdCache( + packageCache: PackageCache, + basedir: string, + originalBasedir: string, +) { + const cacheKey = getFnpdCacheKey(basedir) + const pkgData = packageCache.get(cacheKey) + if (pkgData) { + traverseBetweenDirs(originalBasedir, basedir, (dir) => { + packageCache.set(getFnpdCacheKey(dir), pkgData) + }) + return pkgData } +} - return undefined +function setFnpdCache( + packageCache: PackageCache, + pkgData: PackageData, + basedir: string, + originalBasedir: string, +) { + packageCache.set(getFnpdCacheKey(basedir), pkgData) + traverseBetweenDirs(originalBasedir, basedir, (dir) => { + packageCache.set(getFnpdCacheKey(dir), pkgData) + }) +} + +// package cache key for `findNearestPackageData` +function getFnpdCacheKey(basedir: string) { + return `fnpd_${basedir}` +} + +/** + * Traverse between `longerDir` (inclusive) and `shorterDir` (exclusive) and call `cb` for each dir. + * @param longerDir Longer dir path, e.g. `/User/foo/bar/baz` + * @param shorterDir Shorter dir path, e.g. `/User/foo` + */ +function traverseBetweenDirs( + longerDir: string, + shorterDir: string, + cb: (dir: string) => void, +) { + while (longerDir !== shorterDir) { + cb(longerDir) + longerDir = path.dirname(longerDir) + } } diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index aeb696936530be..9404fdde034ed7 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -36,13 +36,18 @@ import { lookupFile, normalizePath, resolveFrom, + safeRealpathSync, slash, } from '../utils' import { optimizedDepInfoFromFile, optimizedDepInfoFromId } from '../optimizer' import type { DepsOptimizer } from '../optimizer' import type { SSROptions } from '..' import type { PackageCache, PackageData } from '../packages' -import { loadPackageData, resolvePackageData } from '../packages' +import { + findNearestPackageData, + loadPackageData, + resolvePackageData, +} from '../packages' import { isWorkerRequest } from './worker' const normalizedClientEntry = normalizePath(CLIENT_ENTRY) @@ -166,12 +171,9 @@ export function resolvePlugin(resolveOptions: InternalResolveOptions): Plugin { const resolveSubpathImports = (id: string, importer?: string) => { if (!importer || !id.startsWith(subpathImportsPrefix)) return const basedir = path.dirname(importer) - const pkgJsonPath = lookupFile(basedir, ['package.json'], { - pathOnly: true, - }) - if (!pkgJsonPath) return + const pkgData = findNearestPackageData(basedir, options.packageCache) + if (!pkgData) return - const pkgData = loadPackageData(pkgJsonPath, options.preserveSymlinks) let importsPath = resolveExportsOrImports( pkgData.data, id, @@ -183,7 +185,7 @@ export function resolvePlugin(resolveOptions: InternalResolveOptions): Plugin { if (importsPath?.startsWith('.')) { importsPath = path.relative( basedir, - path.join(path.dirname(pkgJsonPath), importsPath), + path.join(pkgData.dir, importsPath), ) if (!importsPath.startsWith('.')) { @@ -608,12 +610,17 @@ function tryResolveFile( return getRealPath(file, options.preserveSymlinks) + postfix } else if (tryIndex) { if (!skipPackageJson) { - const pkgPath = file + '/package.json' + let pkgPath = file + '/package.json' try { - // path points to a node package - const pkg = loadPackageData(pkgPath, options.preserveSymlinks) - const resolved = resolvePackageEntry(file, pkg, targetWeb, options) - return resolved + if (fs.existsSync(pkgPath)) { + if (!options.preserveSymlinks) { + pkgPath = safeRealpathSync(pkgPath) + } + // path points to a node package + const pkg = loadPackageData(pkgPath) + const resolved = resolvePackageEntry(file, pkg, targetWeb, options) + return resolved + } } catch (e) { if (e.code !== 'ENOENT') { throw e @@ -822,7 +829,9 @@ export function tryNodeResolve( (ssr && !( ext === '.cjs' || - (ext === '.js' && resolvePkg(resolved, options)?.data.type !== 'module') + (ext === '.js' && + findNearestPackageData(resolved, options.packageCache)?.data.type !== + 'module') ) && !(include?.includes(pkgId) || include?.includes(id))) @@ -1191,7 +1200,9 @@ function tryResolveBrowserMapping( ) { let res: string | undefined const pkg = - importer && (idToPkgMap.get(importer) || resolvePkg(importer, options)) + importer && + (idToPkgMap.get(importer) || + findNearestPackageData(importer, options.packageCache)) if (pkg && isObject(pkg.data.browser)) { const mapId = isFilePath ? './' + slash(path.relative(pkg.dir, id)) : id const browserMappedPath = mapWithBrowserField(mapId, pkg.data.browser) @@ -1253,23 +1264,3 @@ function getRealPath(resolved: string, preserveSymlinks?: boolean): string { } return normalizePath(resolved) } - -/** - * Load closest `package.json` to `importer` - */ -function resolvePkg(importer: string, options: InternalResolveOptions) { - const { preserveSymlinks, packageCache } = options - - if (importer.includes('\x00')) { - return null - } - - const pkgPath = lookupFile(importer, ['package.json'], { pathOnly: true }) - if (pkgPath) { - const pkg = loadPackageData(pkgPath, preserveSymlinks, packageCache) - idToPkgMap.set(importer, pkg) - return pkg - } - - return undefined -} diff --git a/packages/vite/src/node/ssr/ssrExternal.ts b/packages/vite/src/node/ssr/ssrExternal.ts index f3ecb56f2a7312..f7448c6e1a5a4f 100644 --- a/packages/vite/src/node/ssr/ssrExternal.ts +++ b/packages/vite/src/node/ssr/ssrExternal.ts @@ -13,7 +13,7 @@ import { normalizePath, } from '../utils' import type { Logger, ResolvedConfig } from '..' -import { resolvePkgJsonPath } from '../packages' +import { resolvePackageData } from '../packages' const debug = createDebugger('vite:ssr-external') @@ -257,12 +257,12 @@ function cjsSsrCollectExternals( requireEntry = normalizePath(_require.resolve(id, { paths: [root] })) } catch (e) { // no main entry, but deep imports may be allowed - const pkgPath = resolvePkgJsonPath(id, root) - if (pkgPath) { - if (pkgPath.includes('node_modules')) { + const pkgDir = resolvePackageData(id, root)?.dir + if (pkgDir) { + if (pkgDir.includes('node_modules')) { ssrExternals.add(id) } else { - depsToTrace.add(path.dirname(pkgPath)) + depsToTrace.add(path.dirname(pkgDir)) } continue } @@ -277,9 +277,9 @@ function cjsSsrCollectExternals( } // trace the dependencies of linked packages else if (!esmEntry.includes('node_modules')) { - const pkgPath = resolvePkgJsonPath(id, root) - if (pkgPath) { - depsToTrace.add(path.dirname(pkgPath)) + const pkgDir = resolvePackageData(id, root)?.dir + if (pkgDir) { + depsToTrace.add(pkgDir) } } // has separate esm/require entry, assume require entry is cjs @@ -290,18 +290,11 @@ function cjsSsrCollectExternals( // or are there others like SystemJS / AMD that we'd need to handle? // for now, we'll just leave this as is else if (/\.m?js$/.test(esmEntry)) { - const pkgPath = resolvePkgJsonPath(id, root) - if (!pkgPath) { + const pkg = resolvePackageData(id, root)?.data + if (!pkg) { continue } - const pkgContent = fs.readFileSync(pkgPath, 'utf-8') - - if (!pkgContent) { - continue - } - const pkg = JSON.parse(pkgContent) - if (pkg.type === 'module' || esmEntry.endsWith('.mjs')) { ssrExternals.add(id) continue