Skip to content

Commit abc2b9c

Browse files
bluwybluwy
authoredMar 21, 2023
perf: improve package cache usage (#12512)
·
v6.3.0-beta.0create-vite@4.3.0-beta.0
1 parent 7be0ba5 commit abc2b9c

File tree

5 files changed

+232
-113
lines changed

5 files changed

+232
-113
lines changed
 

‎packages/vite/src/node/optimizer/esbuildDepPlugin.ts

+18-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import path from 'node:path'
22
import type { ImportKind, Plugin } from 'esbuild'
33
import { CSS_LANGS_RE, KNOWN_ASSET_TYPES } from '../constants'
44
import { getDepOptimizationConfig } from '..'
5-
import type { ResolvedConfig } from '..'
5+
import type { PackageCache, ResolvedConfig } from '..'
66
import {
77
flattenId,
88
isBuiltin,
@@ -57,14 +57,24 @@ export function esbuildDepPlugin(
5757
? externalTypes.filter((type) => !extensions?.includes('.' + type))
5858
: externalTypes
5959

60+
// use separate package cache for optimizer as it caches paths around node_modules
61+
// and it's unlikely for the core Vite process to traverse into node_modules again
62+
const esmPackageCache: PackageCache = new Map()
63+
const cjsPackageCache: PackageCache = new Map()
64+
6065
// default resolver which prefers ESM
61-
const _resolve = config.createResolver({ asSrc: false, scan: true })
66+
const _resolve = config.createResolver({
67+
asSrc: false,
68+
scan: true,
69+
packageCache: esmPackageCache,
70+
})
6271

6372
// cjs resolver that prefers Node
6473
const _resolveRequire = config.createResolver({
6574
asSrc: false,
6675
isRequire: true,
6776
scan: true,
77+
packageCache: cjsPackageCache,
6878
})
6979

7080
const resolve = (
@@ -116,6 +126,12 @@ export function esbuildDepPlugin(
116126
return {
117127
name: 'vite:dep-pre-bundle',
118128
setup(build) {
129+
// clear package cache when esbuild is finished
130+
build.onEnd(() => {
131+
esmPackageCache.clear()
132+
cjsPackageCache.clear()
133+
})
134+
119135
// externalize assets and commonly known non-js file types
120136
// See #8459 for more details about this require-import conversion
121137
build.onResolve(

‎packages/vite/src/node/optimizer/index.ts

+5-5
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import {
2626
} from '../utils'
2727
import { transformWithEsbuild } from '../plugins/esbuild'
2828
import { ESBUILD_MODULES_TARGET } from '../constants'
29-
import { resolvePkgJsonPath } from '../packages'
29+
import { resolvePackageData } from '../packages'
3030
import { esbuildCjsExternalPlugin, esbuildDepPlugin } from './esbuildDepPlugin'
3131
import { scanImports } from './scan'
3232
export {
@@ -855,7 +855,7 @@ function createOptimizeDepsIncludeResolver(
855855
// 'foo > bar > baz' => 'foo > bar' & 'baz'
856856
const nestedRoot = id.substring(0, lastArrowIndex).trim()
857857
const nestedPath = id.substring(lastArrowIndex + 1).trim()
858-
const basedir = nestedResolvePkgJsonPath(
858+
const basedir = nestedResolveBasedir(
859859
nestedRoot,
860860
config.root,
861861
config.resolve.preserveSymlinks,
@@ -865,16 +865,16 @@ function createOptimizeDepsIncludeResolver(
865865
}
866866

867867
/**
868-
* Like `resolvePkgJsonPath`, but supports resolving nested package names with '>'
868+
* Continously resolve the basedir of packages separated by '>'
869869
*/
870-
function nestedResolvePkgJsonPath(
870+
function nestedResolveBasedir(
871871
id: string,
872872
basedir: string,
873873
preserveSymlinks = false,
874874
) {
875875
const pkgs = id.split('>').map((pkg) => pkg.trim())
876876
for (const pkg of pkgs) {
877-
basedir = resolvePkgJsonPath(pkg, basedir, preserveSymlinks) || basedir
877+
basedir = resolvePackageData(pkg, basedir, preserveSymlinks)?.dir || basedir
878878
}
879879
return basedir
880880
}

‎packages/vite/src/node/packages.ts

+174-55
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fs from 'node:fs'
22
import path from 'node:path'
33
import { createRequire } from 'node:module'
4-
import { createDebugger, createFilter, safeRealpathSync } from './utils'
4+
import { createFilter, safeRealpathSync } from './utils'
55
import type { ResolvedConfig } from './config'
66
import type { Plugin } from './plugin'
77

@@ -13,11 +13,6 @@ if (process.versions.pnp) {
1313
} catch {}
1414
}
1515

16-
const isDebug = process.env.DEBUG
17-
const debug = createDebugger('vite:resolve-details', {
18-
onlyWhenFocused: true,
19-
})
20-
2116
/** Cache for package.json resolution and package.json contents */
2217
export type PackageCache = Map<string, PackageData>
2318

@@ -56,49 +51,99 @@ export function invalidatePackageData(
5651
}
5752

5853
export function resolvePackageData(
59-
id: string,
54+
pkgName: string,
6055
basedir: string,
6156
preserveSymlinks = false,
6257
packageCache?: PackageCache,
6358
): PackageData | null {
64-
let pkg: PackageData | undefined
65-
let cacheKey: string | undefined
66-
if (packageCache) {
67-
cacheKey = `${id}&${basedir}&${preserveSymlinks}`
68-
if ((pkg = packageCache.get(cacheKey))) {
69-
return pkg
70-
}
59+
if (pnp) {
60+
const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks)
61+
if (packageCache?.has(cacheKey)) return packageCache.get(cacheKey)!
62+
63+
const pkg = pnp.resolveToUnqualified(pkgName, basedir)
64+
if (!pkg) return null
65+
66+
const pkgData = loadPackageData(path.join(pkg, 'package.json'))
67+
packageCache?.set(cacheKey, pkgData)
68+
69+
return pkgData
7170
}
72-
const pkgPath = resolvePkgJsonPath(id, basedir, preserveSymlinks)
73-
if (!pkgPath) return null
74-
try {
75-
pkg = loadPackageData(pkgPath, true, packageCache)
71+
72+
const originalBasedir = basedir
73+
while (basedir) {
7674
if (packageCache) {
77-
packageCache.set(cacheKey!, pkg)
78-
}
79-
return pkg
80-
} catch (e) {
81-
if (e instanceof SyntaxError) {
82-
isDebug && debug(`Parsing failed: ${pkgPath}`)
75+
const cached = getRpdCache(
76+
packageCache,
77+
pkgName,
78+
basedir,
79+
originalBasedir,
80+
preserveSymlinks,
81+
)
82+
if (cached) return cached
8383
}
84-
throw e
84+
85+
const pkg = path.join(basedir, 'node_modules', pkgName, 'package.json')
86+
try {
87+
if (fs.existsSync(pkg)) {
88+
const pkgPath = preserveSymlinks ? pkg : safeRealpathSync(pkg)
89+
const pkgData = loadPackageData(pkgPath)
90+
91+
if (packageCache) {
92+
setRpdCache(
93+
packageCache,
94+
pkgData,
95+
pkgName,
96+
basedir,
97+
originalBasedir,
98+
preserveSymlinks,
99+
)
100+
}
101+
102+
return pkgData
103+
}
104+
} catch {}
105+
106+
const nextBasedir = path.dirname(basedir)
107+
if (nextBasedir === basedir) break
108+
basedir = nextBasedir
85109
}
110+
111+
return null
86112
}
87113

88-
export function loadPackageData(
89-
pkgPath: string,
90-
preserveSymlinks?: boolean,
114+
export function findNearestPackageData(
115+
basedir: string,
91116
packageCache?: PackageCache,
92-
): PackageData {
93-
if (!preserveSymlinks) {
94-
pkgPath = safeRealpathSync(pkgPath)
95-
}
117+
): PackageData | null {
118+
const originalBasedir = basedir
119+
while (basedir) {
120+
if (packageCache) {
121+
const cached = getFnpdCache(packageCache, basedir, originalBasedir)
122+
if (cached) return cached
123+
}
124+
125+
const pkgPath = path.join(basedir, 'package.json')
126+
try {
127+
if (fs.statSync(pkgPath, { throwIfNoEntry: false })?.isFile()) {
128+
const pkgData = loadPackageData(pkgPath)
129+
130+
if (packageCache) {
131+
setFnpdCache(packageCache, pkgData, basedir, originalBasedir)
132+
}
96133

97-
let cached: PackageData | undefined
98-
if ((cached = packageCache?.get(pkgPath))) {
99-
return cached
134+
return pkgData
135+
}
136+
} catch {}
137+
138+
const nextBasedir = path.dirname(basedir)
139+
if (nextBasedir === basedir) break
140+
basedir = nextBasedir
100141
}
101142

143+
return null
144+
}
145+
146+
export function loadPackageData(pkgPath: string): PackageData {
102147
const data = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'))
103148
const pkgDir = path.dirname(pkgPath)
104149
const { sideEffects } = data
@@ -147,7 +192,6 @@ export function loadPackageData(
147192
},
148193
}
149194

150-
packageCache?.set(pkgPath, pkg)
151195
return pkg
152196
}
153197

@@ -184,29 +228,104 @@ export function watchPackageDataPlugin(config: ResolvedConfig): Plugin {
184228
}
185229
}
186230

187-
export function resolvePkgJsonPath(
231+
/**
232+
* Get cached `resolvePackageData` value based on `basedir`. When one is found,
233+
* and we've already traversed some directories between `basedir` and `originalBasedir`,
234+
* we cache the value for those in-between directories as well.
235+
*
236+
* This makes it so the fs is only read once for a shared `basedir`.
237+
*/
238+
function getRpdCache(
239+
packageCache: PackageCache,
188240
pkgName: string,
189241
basedir: string,
190-
preserveSymlinks = false,
191-
): string | undefined {
192-
if (pnp) {
193-
const pkg = pnp.resolveToUnqualified(pkgName, basedir)
194-
if (!pkg) return undefined
195-
return path.join(pkg, 'package.json')
242+
originalBasedir: string,
243+
preserveSymlinks: boolean,
244+
) {
245+
const cacheKey = getRpdCacheKey(pkgName, basedir, preserveSymlinks)
246+
const pkgData = packageCache.get(cacheKey)
247+
if (pkgData) {
248+
traverseBetweenDirs(originalBasedir, basedir, (dir) => {
249+
packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData)
250+
})
251+
return pkgData
196252
}
253+
}
197254

198-
let root = basedir
199-
while (root) {
200-
const pkg = path.join(root, 'node_modules', pkgName, 'package.json')
201-
try {
202-
if (fs.existsSync(pkg)) {
203-
return preserveSymlinks ? pkg : safeRealpathSync(pkg)
204-
}
205-
} catch {}
206-
const nextRoot = path.dirname(root)
207-
if (nextRoot === root) break
208-
root = nextRoot
255+
function setRpdCache(
256+
packageCache: PackageCache,
257+
pkgData: PackageData,
258+
pkgName: string,
259+
basedir: string,
260+
originalBasedir: string,
261+
preserveSymlinks: boolean,
262+
) {
263+
packageCache.set(getRpdCacheKey(pkgName, basedir, preserveSymlinks), pkgData)
264+
traverseBetweenDirs(originalBasedir, basedir, (dir) => {
265+
packageCache.set(getRpdCacheKey(pkgName, dir, preserveSymlinks), pkgData)
266+
})
267+
}
268+
269+
// package cache key for `resolvePackageData`
270+
function getRpdCacheKey(
271+
pkgName: string,
272+
basedir: string,
273+
preserveSymlinks: boolean,
274+
) {
275+
return `rpd_${pkgName}_${basedir}_${preserveSymlinks}`
276+
}
277+
278+
/**
279+
* Get cached `findNearestPackageData` value based on `basedir`. When one is found,
280+
* and we've already traversed some directories between `basedir` and `originalBasedir`,
281+
* we cache the value for those in-between directories as well.
282+
*
283+
* This makes it so the fs is only read once for a shared `basedir`.
284+
*/
285+
function getFnpdCache(
286+
packageCache: PackageCache,
287+
basedir: string,
288+
originalBasedir: string,
289+
) {
290+
const cacheKey = getFnpdCacheKey(basedir)
291+
const pkgData = packageCache.get(cacheKey)
292+
if (pkgData) {
293+
traverseBetweenDirs(originalBasedir, basedir, (dir) => {
294+
packageCache.set(getFnpdCacheKey(dir), pkgData)
295+
})
296+
return pkgData
209297
}
298+
}
210299

211-
return undefined
300+
function setFnpdCache(
301+
packageCache: PackageCache,
302+
pkgData: PackageData,
303+
basedir: string,
304+
originalBasedir: string,
305+
) {
306+
packageCache.set(getFnpdCacheKey(basedir), pkgData)
307+
traverseBetweenDirs(originalBasedir, basedir, (dir) => {
308+
packageCache.set(getFnpdCacheKey(dir), pkgData)
309+
})
310+
}
311+
312+
// package cache key for `findNearestPackageData`
313+
function getFnpdCacheKey(basedir: string) {
314+
return `fnpd_${basedir}`
315+
}
316+
317+
/**
318+
* Traverse between `longerDir` (inclusive) and `shorterDir` (exclusive) and call `cb` for each dir.
319+
* @param longerDir Longer dir path, e.g. `/User/foo/bar/baz`
320+
* @param shorterDir Shorter dir path, e.g. `/User/foo`
321+
*/
322+
function traverseBetweenDirs(
323+
longerDir: string,
324+
shorterDir: string,
325+
cb: (dir: string) => void,
326+
) {
327+
while (longerDir !== shorterDir) {
328+
cb(longerDir)
329+
longerDir = path.dirname(longerDir)
330+
}
212331
}

‎packages/vite/src/node/plugins/resolve.ts

+25-34
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,18 @@ import {
3636
lookupFile,
3737
normalizePath,
3838
resolveFrom,
39+
safeRealpathSync,
3940
slash,
4041
} from '../utils'
4142
import { optimizedDepInfoFromFile, optimizedDepInfoFromId } from '../optimizer'
4243
import type { DepsOptimizer } from '../optimizer'
4344
import type { SSROptions } from '..'
4445
import type { PackageCache, PackageData } from '../packages'
45-
import { loadPackageData, resolvePackageData } from '../packages'
46+
import {
47+
findNearestPackageData,
48+
loadPackageData,
49+
resolvePackageData,
50+
} from '../packages'
4651
import { isWorkerRequest } from './worker'
4752

4853
const normalizedClientEntry = normalizePath(CLIENT_ENTRY)
@@ -166,12 +171,9 @@ export function resolvePlugin(resolveOptions: InternalResolveOptions): Plugin {
166171
const resolveSubpathImports = (id: string, importer?: string) => {
167172
if (!importer || !id.startsWith(subpathImportsPrefix)) return
168173
const basedir = path.dirname(importer)
169-
const pkgJsonPath = lookupFile(basedir, ['package.json'], {
170-
pathOnly: true,
171-
})
172-
if (!pkgJsonPath) return
174+
const pkgData = findNearestPackageData(basedir, options.packageCache)
175+
if (!pkgData) return
173176

174-
const pkgData = loadPackageData(pkgJsonPath, options.preserveSymlinks)
175177
let importsPath = resolveExportsOrImports(
176178
pkgData.data,
177179
id,
@@ -183,7 +185,7 @@ export function resolvePlugin(resolveOptions: InternalResolveOptions): Plugin {
183185
if (importsPath?.startsWith('.')) {
184186
importsPath = path.relative(
185187
basedir,
186-
path.join(path.dirname(pkgJsonPath), importsPath),
188+
path.join(pkgData.dir, importsPath),
187189
)
188190

189191
if (!importsPath.startsWith('.')) {
@@ -608,12 +610,17 @@ function tryResolveFile(
608610
return getRealPath(file, options.preserveSymlinks) + postfix
609611
} else if (tryIndex) {
610612
if (!skipPackageJson) {
611-
const pkgPath = file + '/package.json'
613+
let pkgPath = file + '/package.json'
612614
try {
613-
// path points to a node package
614-
const pkg = loadPackageData(pkgPath, options.preserveSymlinks)
615-
const resolved = resolvePackageEntry(file, pkg, targetWeb, options)
616-
return resolved
615+
if (fs.existsSync(pkgPath)) {
616+
if (!options.preserveSymlinks) {
617+
pkgPath = safeRealpathSync(pkgPath)
618+
}
619+
// path points to a node package
620+
const pkg = loadPackageData(pkgPath)
621+
const resolved = resolvePackageEntry(file, pkg, targetWeb, options)
622+
return resolved
623+
}
617624
} catch (e) {
618625
if (e.code !== 'ENOENT') {
619626
throw e
@@ -822,7 +829,9 @@ export function tryNodeResolve(
822829
(ssr &&
823830
!(
824831
ext === '.cjs' ||
825-
(ext === '.js' && resolvePkg(resolved, options)?.data.type !== 'module')
832+
(ext === '.js' &&
833+
findNearestPackageData(resolved, options.packageCache)?.data.type !==
834+
'module')
826835
) &&
827836
!(include?.includes(pkgId) || include?.includes(id)))
828837

@@ -1191,7 +1200,9 @@ function tryResolveBrowserMapping(
11911200
) {
11921201
let res: string | undefined
11931202
const pkg =
1194-
importer && (idToPkgMap.get(importer) || resolvePkg(importer, options))
1203+
importer &&
1204+
(idToPkgMap.get(importer) ||
1205+
findNearestPackageData(importer, options.packageCache))
11951206
if (pkg && isObject(pkg.data.browser)) {
11961207
const mapId = isFilePath ? './' + slash(path.relative(pkg.dir, id)) : id
11971208
const browserMappedPath = mapWithBrowserField(mapId, pkg.data.browser)
@@ -1253,23 +1264,3 @@ function getRealPath(resolved: string, preserveSymlinks?: boolean): string {
12531264
}
12541265
return normalizePath(resolved)
12551266
}
1256-
1257-
/**
1258-
* Load closest `package.json` to `importer`
1259-
*/
1260-
function resolvePkg(importer: string, options: InternalResolveOptions) {
1261-
const { preserveSymlinks, packageCache } = options
1262-
1263-
if (importer.includes('\x00')) {
1264-
return null
1265-
}
1266-
1267-
const pkgPath = lookupFile(importer, ['package.json'], { pathOnly: true })
1268-
if (pkgPath) {
1269-
const pkg = loadPackageData(pkgPath, preserveSymlinks, packageCache)
1270-
idToPkgMap.set(importer, pkg)
1271-
return pkg
1272-
}
1273-
1274-
return undefined
1275-
}

‎packages/vite/src/node/ssr/ssrExternal.ts

+10-17
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
normalizePath,
1414
} from '../utils'
1515
import type { Logger, ResolvedConfig } from '..'
16-
import { resolvePkgJsonPath } from '../packages'
16+
import { resolvePackageData } from '../packages'
1717

1818
const debug = createDebugger('vite:ssr-external')
1919

@@ -257,12 +257,12 @@ function cjsSsrCollectExternals(
257257
requireEntry = normalizePath(_require.resolve(id, { paths: [root] }))
258258
} catch (e) {
259259
// no main entry, but deep imports may be allowed
260-
const pkgPath = resolvePkgJsonPath(id, root)
261-
if (pkgPath) {
262-
if (pkgPath.includes('node_modules')) {
260+
const pkgDir = resolvePackageData(id, root)?.dir
261+
if (pkgDir) {
262+
if (pkgDir.includes('node_modules')) {
263263
ssrExternals.add(id)
264264
} else {
265-
depsToTrace.add(path.dirname(pkgPath))
265+
depsToTrace.add(path.dirname(pkgDir))
266266
}
267267
continue
268268
}
@@ -277,9 +277,9 @@ function cjsSsrCollectExternals(
277277
}
278278
// trace the dependencies of linked packages
279279
else if (!esmEntry.includes('node_modules')) {
280-
const pkgPath = resolvePkgJsonPath(id, root)
281-
if (pkgPath) {
282-
depsToTrace.add(path.dirname(pkgPath))
280+
const pkgDir = resolvePackageData(id, root)?.dir
281+
if (pkgDir) {
282+
depsToTrace.add(pkgDir)
283283
}
284284
}
285285
// has separate esm/require entry, assume require entry is cjs
@@ -290,18 +290,11 @@ function cjsSsrCollectExternals(
290290
// or are there others like SystemJS / AMD that we'd need to handle?
291291
// for now, we'll just leave this as is
292292
else if (/\.m?js$/.test(esmEntry)) {
293-
const pkgPath = resolvePkgJsonPath(id, root)
294-
if (!pkgPath) {
293+
const pkg = resolvePackageData(id, root)?.data
294+
if (!pkg) {
295295
continue
296296
}
297297

298-
const pkgContent = fs.readFileSync(pkgPath, 'utf-8')
299-
300-
if (!pkgContent) {
301-
continue
302-
}
303-
const pkg = JSON.parse(pkgContent)
304-
305298
if (pkg.type === 'module' || esmEntry.endsWith('.mjs')) {
306299
ssrExternals.add(id)
307300
continue

0 commit comments

Comments
 (0)
Please sign in to comment.