From 0060c114ab7d9e34162cd6677f09be75bde18031 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Fri, 18 Mar 2022 23:06:21 +0100 Subject: [PATCH 01/18] feat: non-blocking scanning of dependencies --- packages/vite/src/node/index.ts | 3 +- .../src/node/optimizer/esbuildDepPlugin.ts | 5 +- packages/vite/src/node/optimizer/index.ts | 716 ++++++++++-------- .../src/node/optimizer/registerMissing.ts | 186 +++-- packages/vite/src/node/optimizer/scan.ts | 5 +- packages/vite/src/node/plugin.ts | 2 +- .../vite/src/node/plugins/importAnalysis.ts | 37 +- .../vite/src/node/plugins/optimizedDeps.ts | 4 +- packages/vite/src/node/plugins/preAlias.ts | 6 +- packages/vite/src/node/plugins/resolve.ts | 75 +- packages/vite/src/node/server/index.ts | 53 +- .../vite/src/node/server/pluginContainer.ts | 12 +- 12 files changed, 650 insertions(+), 454 deletions(-) diff --git a/packages/vite/src/node/index.ts b/packages/vite/src/node/index.ts index 99567735b21757..14af98d99f4893 100644 --- a/packages/vite/src/node/index.ts +++ b/packages/vite/src/node/index.ts @@ -39,7 +39,8 @@ export type { DepOptimizationOptions, DepOptimizationResult, DepOptimizationProcessing, - OptimizedDepInfo + OptimizedDepInfo, + OptimizedDeps } from './optimizer' export type { Plugin } from './plugin' export type { PackageCache, PackageData } from './packages' diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index 3ff86c213a54a2..9ebd59b76b12fd 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -48,12 +48,13 @@ export function esbuildDepPlugin( : externalTypes // default resolver which prefers ESM - const _resolve = config.createResolver({ asSrc: false }) + const _resolve = config.createResolver({ asSrc: false, scan: true }) // cjs resolver that prefers Node const _resolveRequire = config.createResolver({ asSrc: false, - isRequire: true + isRequire: true, + scan: true }) const resolve = ( diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index becc1dd5fad066..5658841639afbf 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -24,12 +24,25 @@ import { performance } from 'perf_hooks' const debug = createDebugger('vite:deps') const isDebugEnabled = _debug('vite:deps').enabled +const jsExtensionRE = /\.js$/i +const jsMapExtensionRE = /\.js\.map$/i + export type ExportsData = ReturnType & { // es-module-lexer has a facade detection but isn't always accurate for our // use case when the module has default export hasReExports?: true } +export type OptimizedDeps = { + metadata: DepOptimizationMetadata + scanProcessing?: Promise + registerMissingImport: ( + id: string, + resolved: string, + ssr?: boolean + ) => OptimizedDepInfo +} + export interface DepOptimizationOptions { /** * By default, Vite will crawl your `index.html` to detect dependencies that @@ -94,6 +107,7 @@ export interface DepOptimizationOptions { } export interface DepOptimizationResult { + metadata: DepOptimizationMetadata /** * After a re-optimization, the internal bundled chunks may change * and a full page reload is required if that is the case @@ -103,8 +117,8 @@ export interface DepOptimizationResult { alteredFiles: boolean /** * When doing a re-run, if there are newly discovered dependendencies - * the page reload will be delayed until the next rerun so the - * result will be discarded + * the page reload will be delayed until the next rerun so we need + * to be able to discard the result */ commit: () => void cancel: () => void @@ -125,7 +139,7 @@ export interface OptimizedDepInfo { * During optimization, ids can still be resolved to their final location * but the bundles may not yet be saved to disk */ - processing: Promise + processing?: Promise } export interface DepOptimizationMetadata { @@ -144,6 +158,10 @@ export interface DepOptimizationMetadata { * Metadata for each already optimized dependency */ optimized: Record + /** + * Metadata for non-entry optimized chunks and dynamic imports + */ + chunks: Record /** * Metadata for each newly discovered dependency after processing */ @@ -156,45 +174,49 @@ export interface DepOptimizationMetadata { export async function optimizeDeps( config: ResolvedConfig, force = config.server.force, - asCommand = false, - newDeps?: Record, // missing imports encountered after server has started - ssr?: boolean + asCommand = false ): Promise { - const { metadata, run } = await createOptimizeDepsRun( + const cachedMetadata = loadCachedDepOptimizationMetadata( config, force, - asCommand, - null, - newDeps, - ssr + asCommand ) - const result = await run() + if (cachedMetadata) { + return cachedMetadata + } + const depsInfo = await discoverProjectDependencies(config) + + const depsString = depsLogString(config, Object.keys(depsInfo)) + config.logger.info(colors.green(`Optimizing dependencies:\n ${depsString}`)) + + const result = await createOptimizeDepsRun(config, depsInfo) + result.commit() - return metadata + + return result.metadata +} + +export function createOptimizedDepsMetadata(config: ResolvedConfig) { + const mainHash = getDepHash(config) + return { + hash: mainHash, + browserHash: mainHash, + optimized: {}, + chunks: {}, + discovered: {} + } } /** - * Internally, Vite uses this function to prepare a optimizeDeps run. When Vite starts, we can get - * the metadata and start the server without waiting for the optimizeDeps processing to be completed + * Creates the initial dep optimization metadata, loading it from the deps cache + * if it exists and pre-bundling isn't forced */ -export async function createOptimizeDepsRun( +export function loadCachedDepOptimizationMetadata( config: ResolvedConfig, force = config.server.force, - asCommand = false, - currentData: DepOptimizationMetadata | null = null, - newDeps?: Record, // missing imports encountered after server has started - ssr?: boolean -): Promise<{ - metadata: DepOptimizationMetadata - run: () => Promise -}> { - config = { - ...config, - command: 'build' - } - - const { root, logger } = config - const log = asCommand ? logger.info : debug + asCommand = false +): DepOptimizationMetadata | undefined { + const log = asCommand ? config.logger.info : debug // Before Vite 2.9, dependencies were cached in the root of the cacheDir // For compat, we remove the cache if we find the old structure @@ -203,50 +225,110 @@ export async function createOptimizeDepsRun( } const depsCacheDir = getDepsCacheDir(config) - const processingCacheDir = getProcessingDepsCacheDir(config) - - const mainHash = getDepHash(root, config) - const processing = newDepOptimizationProcessing() - - const metadata: DepOptimizationMetadata = { - hash: mainHash, - browserHash: mainHash, - optimized: {}, - discovered: {} - } + const mainHash = getDepHash(config) if (!force) { - let prevData: DepOptimizationMetadata | undefined + let cachedMetadata: DepOptimizationMetadata | undefined try { - const prevDataPath = path.join(depsCacheDir, '_metadata.json') - prevData = parseOptimizedDepsMetadata( - fs.readFileSync(prevDataPath, 'utf-8'), - depsCacheDir, - processing.promise + const cachedMetadataPatah = path.join(depsCacheDir, '_metadata.json') + cachedMetadata = parseOptimizedDepsMetadata( + fs.readFileSync(cachedMetadataPatah, 'utf-8'), + depsCacheDir ) } catch (e) {} // hash is consistent, no need to re-bundle - if (prevData && prevData.hash === metadata.hash) { + if (cachedMetadata && cachedMetadata.hash === mainHash) { log('Hash is consistent. Skipping. Use --force to override.') // Nothing to commit or cancel as we are using the cache, we only // need to resolve the processing promise so requests can move on - const resolve = () => { - processing.resolve() - } - return { - metadata: prevData, - run: async () => { - return { - alteredFiles: false, - commit: resolve, - cancel: resolve - } - } - } + return cachedMetadata } } + // Start with a fresh cache + removeDirSync(depsCacheDir) +} + +/** + * Initial optimizeDeps at server start. Perform a fast scan using esbuild to + * find deps to pre-bundle and include user hard-coded dependencies + */ + +export async function discoverProjectDependencies( + config: ResolvedConfig +): Promise> { + const { deps, missing } = await scanImports(config) + + const missingIds = Object.keys(missing) + if (missingIds.length) { + throw new Error( + `The following dependencies are imported but could not be resolved:\n\n ${missingIds + .map( + (id) => + `${colors.cyan(id)} ${colors.white( + colors.dim(`(imported by ${missing[id]})`) + )}` + ) + .join(`\n `)}\n\nAre they installed?` + ) + } + + await addManuallyIncludedOptimizeDeps(deps, config) + + const browserHash = getOptimizedBrowserHash(getDepHash(config), deps) + const discovered: Record = {} + for (const id in deps) { + const entry = deps[id] + discovered[id] = { + file: getOptimizedDepPath(id, config), + src: entry, + browserHash: browserHash + } + } + return discovered +} + +export function depsLogString( + config: ResolvedConfig, + qualifiedIds: string[] +): string { + let depsString: string + if (isDebugEnabled) { + depsString = colors.yellow(qualifiedIds.join(`\n `)) + } else { + const total = qualifiedIds.length + const maxListed = 5 + const listed = Math.min(total, maxListed) + const extra = Math.max(0, total - maxListed) + depsString = colors.yellow( + qualifiedIds.slice(0, listed).join(`, `) + + (extra > 0 ? `, ...and ${extra} more` : ``) + ) + } + return depsString +} + +/** + * Internally, Vite uses this function to prepare a optimizeDeps run. When Vite starts, we can get + * the metadata and start the server without waiting for the optimizeDeps processing to be completed + */ +export async function createOptimizeDepsRun( + config: ResolvedConfig, + depsInfo: Record, + currentData?: DepOptimizationMetadata, + ssr?: boolean +): Promise { + config = { + ...config, + command: 'build' + } + + const depsCacheDir = getDepsCacheDir(config) + const processingCacheDir = getProcessingDepsCacheDir(config) + + const mainHash = getDepHash(config) + // Create a temporal directory so we don't need to delete optimized deps // until they have been processed. This also avoids leaving the deps cache // directory in a corrupted state if there is an error @@ -263,282 +345,217 @@ export async function createOptimizeDepsRun( JSON.stringify({ type: 'module' }) ) - let newBrowserHash: string - - let deps: Record - if (!newDeps) { - // Initial optimizeDeps at server start. Perform a fast scan using esbuild to - // find deps to pre-bundle and include user hard-coded dependencies - - let missing: Record - ;({ deps, missing } = await scanImports(config)) - - const missingIds = Object.keys(missing) - if (missingIds.length) { - processing.resolve() - throw new Error( - `The following dependencies are imported but could not be resolved:\n\n ${missingIds - .map( - (id) => - `${colors.cyan(id)} ${colors.white( - colors.dim(`(imported by ${missing[id]})`) - )}` - ) - .join(`\n `)}\n\nAre they installed?` - ) - } + const deps = depsFromOptimizedDepInfo(depsInfo) - try { - await addManuallyIncludedOptimizeDeps(deps, config) - } catch (e) { - processing.resolve() - throw e - } - - // update browser hash - newBrowserHash = metadata.browserHash = getOptimizedBrowserHash( - metadata.hash, - deps - ) - - // We generate the mapping of dependency ids to their cache file location - // before processing the dependencies with esbuild. This allow us to continue - // processing files in the importAnalysis and resolve plugins - for (const id in deps) { - const entry = deps[id] - metadata.optimized[id] = { - file: getOptimizedDepPath(id, config), - src: entry, - browserHash: newBrowserHash, - processing: processing.promise - } - } - } else { - // Missing dependencies were found at run-time, optimizeDeps called while the - // server is running - deps = depsFromOptimizedDepInfo(newDeps) - - metadata.optimized = newDeps + const newBrowserHash = getOptimizedBrowserHash(mainHash, deps) + const metadata: DepOptimizationMetadata = { + hash: mainHash, // For reruns keep current global browser hash and newDeps individual hashes until we know // if files are stable so we can avoid a full page reload - metadata.browserHash = currentData!.browserHash - newBrowserHash = getOptimizedBrowserHash(metadata.hash, deps) + browserHash: currentData?.browserHash || newBrowserHash, + optimized: depsInfo, + chunks: {}, + discovered: {} } - return { metadata, run: prebundleDeps } - - async function prebundleDeps(): Promise { - // We prebundle dependencies with esbuild and cache them, but there is no need - // to wait here. Code that needs to access the cached deps needs to await - // the optimizeDepInfo.processing promise for each dep - - const qualifiedIds = Object.keys(deps) - - if (!qualifiedIds.length) { - return { - alteredFiles: false, - commit() { - // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` - commitProcessingDepsCacheSync() - log(`No dependencies to bundle. Skipping.\n\n\n`) - processing.resolve() - }, - cancel - } - } + // We prebundle dependencies with esbuild and cache them, but there is no need + // to wait here. Code that needs to access the cached deps needs to await + // the optimizeDepInfo.processing promise for each dep - let depsString: string - if (isDebugEnabled) { - depsString = colors.yellow(qualifiedIds.join(`\n `)) - } else { - const total = qualifiedIds.length - const maxListed = 5 - const listed = Math.min(total, maxListed) - const extra = Math.max(0, total - maxListed) - depsString = colors.yellow( - qualifiedIds.slice(0, listed).join(`\n `) + - (extra > 0 ? `\n (...and ${extra} more)` : ``) - ) - } + const qualifiedIds = Object.keys(deps) - if (!asCommand) { - if (!newDeps) { - // This is auto run on server start - let the user know that we are - // pre-optimizing deps - logger.info(colors.green(`Pre-bundling dependencies:\n ${depsString}`)) - logger.info( - `(this will be run only when your dependencies or config have changed)` - ) - } - } else { - logger.info(colors.green(`Optimizing dependencies:\n ${depsString}`)) + if (!qualifiedIds.length) { + return { + metadata, + alteredFiles: false, + commit() { + // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` + commitProcessingDepsCacheSync() + config.logger.info(`No dependencies to bundle. Skipping.\n\n\n`) + }, + cancel } + } - // esbuild generates nested directory output with lowest common ancestor base - // this is unpredictable and makes it difficult to analyze entry / output - // mapping. So what we do here is: - // 1. flatten all ids to eliminate slash - // 2. in the plugin, read the entry ourselves as virtual files to retain the - // path. - const flatIdDeps: Record = {} - const idToExports: Record = {} - const flatIdToExports: Record = {} - - const { plugins = [], ...esbuildOptions } = - config.optimizeDeps?.esbuildOptions ?? {} - - await init - for (const id in deps) { - const flatId = flattenId(id) - const filePath = (flatIdDeps[flatId] = deps[id]) - let exportsData: ExportsData - if ( - config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext)) - ) { - // For custom supported extensions, build the entry file to transform it into JS, - // and then parse with es-module-lexer. Note that the `bundle` option is not `true`, - // so only the entry file is being transformed. - const result = await build({ - ...esbuildOptions, - plugins, - entryPoints: [filePath], - write: false, - format: 'esm' + // esbuild generates nested directory output with lowest common ancestor base + // this is unpredictable and makes it difficult to analyze entry / output + // mapping. So what we do here is: + // 1. flatten all ids to eliminate slash + // 2. in the plugin, read the entry ourselves as virtual files to retain the + // path. + const flatIdDeps: Record = {} + const idToExports: Record = {} + const flatIdToExports: Record = {} + + const { plugins = [], ...esbuildOptions } = + config.optimizeDeps?.esbuildOptions ?? {} + + await init + for (const id in deps) { + const flatId = flattenId(id) + const filePath = (flatIdDeps[flatId] = deps[id]) + let exportsData: ExportsData + if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) { + // For custom supported extensions, build the entry file to transform it into JS, + // and then parse with es-module-lexer. Note that the `bundle` option is not `true`, + // so only the entry file is being transformed. + const result = await build({ + ...esbuildOptions, + plugins, + entryPoints: [filePath], + write: false, + format: 'esm' + }) + exportsData = parse(result.outputFiles[0].text) as ExportsData + } else { + const entryContent = fs.readFileSync(filePath, 'utf-8') + try { + exportsData = parse(entryContent) as ExportsData + } catch { + debug( + `Unable to parse dependency: ${id}. Trying again with a JSX transform.` + ) + const transformed = await transformWithEsbuild(entryContent, filePath, { + loader: 'jsx' }) - exportsData = parse(result.outputFiles[0].text) as ExportsData - } else { - const entryContent = fs.readFileSync(filePath, 'utf-8') - try { - exportsData = parse(entryContent) as ExportsData - } catch { - debug( - `Unable to parse dependency: ${id}. Trying again with a JSX transform.` - ) - const transformed = await transformWithEsbuild( - entryContent, - filePath, - { - loader: 'jsx' - } - ) - // Ensure that optimization won't fail by defaulting '.js' to the JSX parser. - // This is useful for packages such as Gatsby. - esbuildOptions.loader = { - '.js': 'jsx', - ...esbuildOptions.loader - } - exportsData = parse(transformed.code) as ExportsData + // Ensure that optimization won't fail by defaulting '.js' to the JSX parser. + // This is useful for packages such as Gatsby. + esbuildOptions.loader = { + '.js': 'jsx', + ...esbuildOptions.loader } - for (const { ss, se } of exportsData[0]) { - const exp = entryContent.slice(ss, se) - if (/export\s+\*\s+from/.test(exp)) { - exportsData.hasReExports = true - } + exportsData = parse(transformed.code) as ExportsData + } + for (const { ss, se } of exportsData[0]) { + const exp = entryContent.slice(ss, se) + if (/export\s+\*\s+from/.test(exp)) { + exportsData.hasReExports = true } } - - idToExports[id] = exportsData - flatIdToExports[flatId] = exportsData } - const define: Record = { - 'process.env.NODE_ENV': JSON.stringify(config.mode) - } - for (const key in config.define) { - const value = config.define[key] - define[key] = typeof value === 'string' ? value : JSON.stringify(value) - } + idToExports[id] = exportsData + flatIdToExports[flatId] = exportsData + } - const start = performance.now() - - const result = await build({ - absWorkingDir: process.cwd(), - entryPoints: Object.keys(flatIdDeps), - bundle: true, - format: 'esm', - target: config.build.target || undefined, - external: config.optimizeDeps?.exclude, - logLevel: 'error', - splitting: true, - sourcemap: true, - outdir: processingCacheDir, - ignoreAnnotations: true, - metafile: true, - define, - plugins: [ - ...plugins, - esbuildDepPlugin(flatIdDeps, flatIdToExports, config, ssr) - ], - ...esbuildOptions - }) + const define: Record = { + 'process.env.NODE_ENV': JSON.stringify(config.mode) + } + for (const key in config.define) { + const value = config.define[key] + define[key] = typeof value === 'string' ? value : JSON.stringify(value) + } - const meta = result.metafile! + const start = performance.now() + + const result = await build({ + absWorkingDir: process.cwd(), + entryPoints: Object.keys(flatIdDeps), + bundle: true, + format: 'esm', + target: config.build.target || undefined, + external: config.optimizeDeps?.exclude, + logLevel: 'error', + splitting: true, + sourcemap: true, + outdir: processingCacheDir, + ignoreAnnotations: true, + metafile: true, + define, + plugins: [ + ...plugins, + esbuildDepPlugin(flatIdDeps, flatIdToExports, config, ssr) + ], + ...esbuildOptions + }) - // the paths in `meta.outputs` are relative to `process.cwd()` - const processingCacheDirOutputPath = path.relative( - process.cwd(), - processingCacheDir - ) + const meta = result.metafile! - for (const id in deps) { - const optimizedInfo = metadata.optimized[id] - optimizedInfo.needsInterop = needsInterop( - id, - idToExports[id], - meta.outputs, - processingCacheDirOutputPath + // the paths in `meta.outputs` are relative to `process.cwd()` + const processingCacheDirOutputPath = path.relative( + process.cwd(), + processingCacheDir + ) + + for (const id in deps) { + const optimizedInfo = metadata.optimized[id] + optimizedInfo.needsInterop = needsInterop( + id, + idToExports[id], + meta.outputs, + processingCacheDirOutputPath + ) + const output = + meta.outputs[ + path.relative(process.cwd(), getProcessingDepPath(id, config)) + ] + if (output) { + // We only need to hash the output.imports in to check for stability, but adding the hash + // and file path gives us a unique hash that may be useful for other things in the future + optimizedInfo.fileHash = getHash( + metadata.hash + optimizedInfo.file + JSON.stringify(output.imports) ) - const output = - meta.outputs[path.relative(process.cwd(), optimizedInfo.file)] - if (output) { - // We only need to hash the output.imports in to check for stability, but adding the hash - // and file path gives us a unique hash that may be useful for other things in the future - optimizedInfo.fileHash = getHash( - metadata.hash + optimizedInfo.file + JSON.stringify(output.imports) - ) - } } + } - // This only runs when missing deps are processed. Previous optimized deps are stable if - // the newly discovered deps don't have common chunks with them. Comparing their fileHash we - // can find out if it is safe to keep the current browser state. If one of the file hashes - // changed, a full page reload is needed - let alteredFiles = false - if (currentData) { - alteredFiles = Object.keys(currentData.optimized).some((dep) => { - const currentInfo = currentData.optimized[dep] - const info = metadata.optimized[dep] - return ( - !info?.fileHash || - !currentInfo?.fileHash || - info?.fileHash !== currentInfo?.fileHash - ) - }) - debug(`optimized deps have altered files: ${alteredFiles}`) - } + // This only runs when missing deps are processed. Previous optimized deps are stable if + // the newly discovered deps don't have common chunks with them. Comparing their fileHash we + // can find out if it is safe to keep the current browser state. If one of the file hashes + // changed, a full page reload is needed + let alteredFiles = false + if (currentData) { + alteredFiles = Object.keys(currentData.optimized).some((dep) => { + const currentInfo = currentData.optimized[dep] + const info = metadata.optimized[dep] + return ( + !info?.fileHash || + !currentInfo?.fileHash || + info?.fileHash !== currentInfo?.fileHash + ) + }) + debug(`optimized deps have altered files: ${alteredFiles}`) + } - if (alteredFiles) { - // Overrite individual hashes with the new global browserHash, a full page reload is required - // New deps that ended up with a different hash replaced while doing analysis import are going to - // return a not found so the browser doesn't cache them. And will properly get loaded after the reload - for (const id in deps) { - metadata.optimized[id].browserHash = newBrowserHash + for (const o of Object.keys(meta.outputs)) { + if (!o.match(jsMapExtensionRE)) { + const id = path + .relative(processingCacheDirOutputPath, o) + .replace(jsExtensionRE, '') + const file = getOptimizedDepPath(id, config) + if (!findFileInfo(metadata.optimized, file)) { + metadata.chunks[id] = { + file, + src: '', + needsInterop: false, + browserHash: + (!alteredFiles && currentData?.chunks[id]?.browserHash) || + newBrowserHash + } } - metadata.browserHash = newBrowserHash } + } - debug(`deps bundled in ${(performance.now() - start).toFixed(2)}ms`) - - return { - alteredFiles, - commit() { - // Write metadata file, delete `deps` folder and rename the new `processing` folder to `deps` in sync - commitProcessingDepsCacheSync() - processing.resolve() - }, - cancel + if (alteredFiles) { + // Overwrite individual hashes with the new global browserHash, a full page reload is required + // New deps that ended up with a different hash replaced while doing analysis import are going to + // return a not found so the browser doesn't cache them. And will properly get loaded after the reload + for (const id in deps) { + metadata.optimized[id].browserHash = newBrowserHash } + metadata.browserHash = newBrowserHash + } + + debug(`deps bundled in ${(performance.now() - start).toFixed(2)}ms`) + + return { + metadata, + alteredFiles, + commit() { + // Write metadata file, delete `deps` folder and rename the new `processing` folder to `deps` in sync + commitProcessingDepsCacheSync() + }, + cancel } function commitProcessingDepsCacheSync() { @@ -552,7 +569,6 @@ export async function createOptimizeDepsRun( function cancel() { removeDirSync(processingCacheDir) - processing.resolve() } } @@ -613,19 +629,15 @@ export function depsFromOptimizedDepInfo( ) } -function getHash(text: string) { +export function getHash(text: string) { return createHash('sha256').update(text).digest('hex').substring(0, 8) } export function getOptimizedBrowserHash( hash: string, - deps: Record, - missing?: Record + deps: Record ) { - // update browser hash - return getHash( - hash + JSON.stringify(deps) + (missing ? JSON.stringify(missing) : '') - ) + return getHash(hash + JSON.stringify(deps)) } function getCachedDepFilePath(id: string, depsCacheDir: string) { @@ -640,7 +652,15 @@ export function getDepsCacheDir(config: ResolvedConfig) { return normalizePath(path.resolve(config.cacheDir, 'deps')) } -export function getProcessingDepsCacheDir(config: ResolvedConfig) { +function getProcessingDepFilePath(id: string, processingCacheDir: string) { + return normalizePath(path.resolve(processingCacheDir, flattenId(id) + '.js')) +} + +function getProcessingDepPath(id: string, config: ResolvedConfig) { + return getProcessingDepFilePath(id, getProcessingDepsCacheDir(config)) +} + +function getProcessingDepsCacheDir(config: ResolvedConfig) { return normalizePath(path.resolve(config.cacheDir, 'processing')) } @@ -669,8 +689,7 @@ export function createIsOptimizedDepUrl(config: ResolvedConfig) { function parseOptimizedDepsMetadata( jsonMetadata: string, - depsCacheDir: string, - processing: Promise + depsCacheDir: string ) { const metadata = JSON.parse(jsonMetadata, (key: string, value: string) => { // Paths can be absolute or relative to the deps cache dir where @@ -680,12 +699,27 @@ function parseOptimizedDepsMetadata( } return value }) + const { browserHash } = metadata for (const o of Object.keys(metadata.optimized)) { - metadata.optimized[o].processing = processing + const depInfo = metadata.optimized[o] + depInfo.browserHash = browserHash } - return { ...metadata, discovered: {} } + metadata.chunks ||= {} // Support missing chunks for back compat + for (const o of Object.keys(metadata.chunks)) { + const depInfo = metadata.chunks[o] + depInfo.src = '' + depInfo.browserHash = browserHash + } + metadata.discovered = {} + return metadata } +/** + * Stringify metadata for deps cache. Remove processing promises + * and individual dep info browserHash. Once the cache is reload + * the next time the server start we need to use the global + * browserHash to allow long term caching + */ function stringifyOptimizedDepsMetadata( metadata: DepOptimizationMetadata, depsCacheDir: string @@ -693,12 +727,41 @@ function stringifyOptimizedDepsMetadata( return JSON.stringify( metadata, (key: string, value: any) => { - if (key === 'processing' || key === 'discovered') { + if (key === 'discovered' || key === 'processing') { return } if (key === 'file' || key === 'src') { return normalizePath(path.relative(depsCacheDir, value)) } + if (key === 'optimized') { + // Only remove browserHash for individual dep info + const cleaned: Record = {} + for (const dep of Object.keys(value)) { + const { browserHash, ...c } = value[dep] + cleaned[dep] = c + } + return cleaned + } + if (key === 'optimized') { + return Object.keys(value).reduce( + (cleaned: Record, dep: string) => { + const { browserHash, ...c } = value[dep] + cleaned[dep] = c + return cleaned + }, + {} + ) + } + if (key === 'chunks') { + return Object.keys(value).reduce( + (cleaned: Record, dep: string) => { + const { browserHash, needsInterop, src, ...c } = value[dep] + cleaned[dep] = c + return cleaned + }, + {} + ) + } return value }, 2 @@ -755,8 +818,8 @@ function isSingleDefaultExport(exports: readonly string[]) { const lockfileFormats = ['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'] -function getDepHash(root: string, config: ResolvedConfig): string { - let content = lookupFile(root, lockfileFormats) || '' +function getDepHash(config: ResolvedConfig): string { + let content = lookupFile(config.root, lockfileFormats) || '' // also take config into account // only a subset of config options that can affect dep optimization content += JSON.stringify( @@ -795,6 +858,7 @@ export function optimizeDepInfoFromFile( ): OptimizedDepInfo | undefined { return ( findFileInfo(metadata.optimized, file) || + findFileInfo(metadata.chunks, file) || findFileInfo(metadata.discovered, file) ) } diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 2b29d1c6a9c594..ebe7f97181444f 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -2,14 +2,19 @@ import colors from 'picocolors' import { createOptimizeDepsRun, getOptimizedDepPath, - getOptimizedBrowserHash, + getHash, depsFromOptimizedDepInfo, - newDepOptimizationProcessing + newDepOptimizationProcessing, + loadCachedDepOptimizationMetadata, + createOptimizedDepsMetadata, + discoverProjectDependencies, + depsLogString } from '.' import type { DepOptimizationMetadata, DepOptimizationProcessing, - OptimizedDepInfo + OptimizedDepInfo, + OptimizedDeps } from '.' import type { ViteDevServer } from '..' import { resolveSSRExternal } from '../ssr/ssrExternal' @@ -20,12 +25,19 @@ import { resolveSSRExternal } from '../ssr/ssrExternal' */ const debounceMs = 100 -export function createMissingImporterRegisterFn( +export async function createOptimizedDeps( server: ViteDevServer, - initialProcessingPromise: Promise -): (id: string, resolved: string, ssr?: boolean) => OptimizedDepInfo { - const { logger } = server.config - let metadata = server._optimizeDepsMetadata! + asCommand = false +): Promise { + const { config } = server + const { logger } = config + + const cachedMetadata = loadCachedDepOptimizationMetadata(config) + + const optimizedDeps = { + metadata: cachedMetadata || createOptimizedDepsMetadata(config), + registerMissingImport + } as OptimizedDeps let handle: NodeJS.Timeout | undefined let newDepsDiscovered = false @@ -41,40 +53,74 @@ export function createMissingImporterRegisterFn( } let enqueuedRerun: (() => void) | undefined - let currentlyProcessing = true - initialProcessingPromise.then(() => { - currentlyProcessing = false - enqueuedRerun?.() - }) + let currentlyProcessing = false - async function rerun(ssr: boolean | undefined) { - // debounce time to wait for new missing deps finished, issue a new - // optimization of deps (both old and newly found) once the previous - // optimizeDeps processing is finished + // If there wasn't a cache or it is outdated, perform a fast scan with esbuild + // to quickly find project dependencies and do a first optimize run + if (!cachedMetadata) { + currentlyProcessing = true + + const scanPhaseProcessing = newDepOptimizationProcessing() + optimizedDeps.scanProcessing = scanPhaseProcessing.promise + + const warmUp = async function () { + try { + logger.info(colors.green(`scanning for dependencies...`), { + timestamp: true + }) + + const { metadata } = optimizedDeps + + const discovered = await discoverProjectDependencies(config) + + // Respect the scan phase discover order to improve reproducibility + for (const dep of Object.keys(discovered)) { + discovered[dep].processing = depOptimizationProcessing.promise + } + + // This is auto run on server start - let the user know that we are + // pre-optimizing deps + const depsString = depsLogString(config, Object.keys(discovered)) + logger.info(colors.green(`dependencies found: ${depsString}`), { + timestamp: true + }) + + metadata.discovered = discovered + + scanPhaseProcessing.resolve() + optimizedDeps.scanProcessing = undefined + + runOptimizer() + } catch (e) { + logger.error(e.message) + if (optimizedDeps.scanProcessing) { + scanPhaseProcessing.resolve() + optimizedDeps.scanProcessing = undefined + } + } + } + + setTimeout(warmUp, 0) + } + + async function runOptimizer(ssr?: boolean) { + // Ensure that rerun is called sequentially + enqueuedRerun = undefined + currentlyProcessing = true + + // Ensure that a rerun will not be issued for current discovered deps + if (handle) clearTimeout(handle) // a succesful completion of the optimizeDeps rerun will end up // creating new bundled version of all current and discovered deps // in the cache dir and a new metadata info object assigned - // to server._optimizeDepsMetadata. A fullReload is only issued if + // to optimizeDeps.metadata. A fullReload is only issued if // the previous bundled dependencies have changed. - // if the rerun fails, server._optimizeDepsMetadata remains untouched, + // if the rerun fails, optimizeDeps.metadata remains untouched, // current discovered deps are cleaned, and a fullReload is issued - // Ensure that rerun is called sequentially - enqueuedRerun = undefined - currentlyProcessing = true - - logger.info( - colors.yellow( - `new dependencies found: ${Object.keys(metadata.discovered).join( - ', ' - )}, updating...` - ), - { - timestamp: true - } - ) + let { metadata } = optimizedDeps // All deps, previous known and newly discovered are rebundled, // respect insertion order to keep the metadata file stable @@ -85,9 +131,10 @@ export function createMissingImporterRegisterFn( for (const dep of Object.keys(metadata.optimized)) { newDeps[dep] = { ...metadata.optimized[dep] } } - // Don't clone discovered info objects, they are read after awaited for (const dep of Object.keys(metadata.discovered)) { - newDeps[dep] = metadata.discovered[dep] + // Clone the discovered info discarding its processing promise + const { processing, ...info } = metadata.discovered[dep] + newDeps[dep] = info } newDepsDiscovered = false @@ -103,21 +150,17 @@ export function createMissingImporterRegisterFn( let newData: DepOptimizationMetadata | null = null try { - const optimizeDeps = await createOptimizeDepsRun( - server.config, - true, - false, - metadata, + const processingResult = await createOptimizeDepsRun( + config, newDeps, + metadata, ssr ) - const processingResult = await optimizeDeps.run() - const commitProcessing = () => { processingResult.commit() - newData = optimizeDeps.metadata + newData = processingResult.metadata // update ssr externals if (ssr) { @@ -134,7 +177,21 @@ export function createMissingImporterRegisterFn( newData.discovered[o] = metadata.discovered[o] } } - metadata = server._optimizeDepsMetadata = newData + + // Commit hash and needsInterop changes to the discovered deps info + // object. Allow for code to await for the discovered processing promise + // and use the information in the same object + for (const o of Object.keys(newData.optimized)) { + const discovered = metadata.discovered[o] + if (discovered) { + const optimized = newData.optimized[o] + discovered.browserHash = optimized.browserHash + discovered.fileHash = optimized.fileHash + discovered.needsInterop = optimized.needsInterop + } + } + + metadata = optimizedDeps.metadata = newData resolveEnqueuedProcessingPromises() } @@ -142,7 +199,7 @@ export function createMissingImporterRegisterFn( if (!processingResult.alteredFiles) { commitProcessing() - logger.info(colors.green(`✨ new dependencies pre-bundled...`), { + logger.info(colors.green(`✨ dependencies pre-bundled...`), { timestamp: true }) } else { @@ -202,11 +259,44 @@ export function createMissingImporterRegisterFn( }) } - return function registerMissingImport( + async function rerun(ssr?: boolean) { + // debounce time to wait for new missing deps finished, issue a new + // optimization of deps (both old and newly found) once the previous + // optimizeDeps processing is finished + const deps = Object.keys(optimizedDeps.metadata.discovered) + const depsString = depsLogString(config, deps) + logger.info(colors.green(`new dependencies found: ${depsString}`), { + timestamp: true + }) + runOptimizer(ssr) + } + + const optimizedDepsTimestamp = Date.now() + + function getDiscoveredBrowserHash( + hash: string, + deps: Record, + missing: Record + ) { + return getHash( + hash + + JSON.stringify(deps) + + JSON.stringify(missing) + + optimizedDepsTimestamp + ) + } + + function registerMissingImport( id: string, resolved: string, ssr?: boolean ): OptimizedDepInfo { + if (optimizedDeps.scanProcessing) { + config.logger.error( + 'Vite internal error: registering missing import before initial scanning is over' + ) + } + const { metadata } = optimizedDeps const optimized = metadata.optimized[id] if (optimized) { return optimized @@ -225,7 +315,7 @@ export function createMissingImporterRegisterFn( // the current state of known + missing deps. If its optimizeDeps run // doesn't alter the bundled files of previous known dependendencies, // we don't need a full reload and this browserHash will be kept - browserHash: getOptimizedBrowserHash( + browserHash: getDiscoveredBrowserHash( metadata.hash, depsFromOptimizedDepInfo(metadata.optimized), depsFromOptimizedDepInfo(metadata.discovered) @@ -251,4 +341,6 @@ export function createMissingImporterRegisterFn( // esbuild is run to generate the pre-bundle return missing } + + return optimizedDeps } diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index ec936534ba644a..d0da3664e98bdd 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -165,7 +165,10 @@ function esbuildScanPlugin( } const resolved = await container.resolveId( id, - importer && normalizePath(importer) + importer && normalizePath(importer), + { + scan: true + } ) const res = resolved?.id seen.set(key, res) diff --git a/packages/vite/src/node/plugin.ts b/packages/vite/src/node/plugin.ts index 36674e242bd33e..1d3cfb2d3bfcdc 100644 --- a/packages/vite/src/node/plugin.ts +++ b/packages/vite/src/node/plugin.ts @@ -121,7 +121,7 @@ export interface Plugin extends RollupPlugin { this: PluginContext, source: string, importer: string | undefined, - options: { custom?: CustomPluginOptions; ssr?: boolean } + options: { custom?: CustomPluginOptions; ssr?: boolean; scan?: boolean } ): Promise | ResolveIdResult load?( this: PluginContext, diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts index b906981b877493..1dc213b2b42f61 100644 --- a/packages/vite/src/node/plugins/importAnalysis.ts +++ b/packages/vite/src/node/plugins/importAnalysis.ts @@ -197,19 +197,22 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { } let importerFile = importer - if ( - moduleListContains(config.optimizeDeps?.exclude, url) && - server._optimizeDepsMetadata - ) { - // if the dependency encountered in the optimized file was excluded from the optimization - // the dependency needs to be resolved starting from the original source location of the optimized file - // because starting from node_modules/.vite will not find the dependency if it was not hoisted - // (that is, if it is under node_modules directory in the package source of the optimized file) - for (const optimizedModule of Object.values( - server._optimizeDepsMetadata.optimized - )) { - if (optimizedModule.file === importerModule.file) { - importerFile = optimizedModule.src + if (moduleListContains(config.optimizeDeps?.exclude, url)) { + const optimizedDeps = server._optimizedDeps + if (optimizedDeps) { + await optimizedDeps.scanProcessing + + // if the dependency encountered in the optimized file was excluded from the optimization + // the dependency needs to be resolved starting from the original source location of the optimized file + // because starting from node_modules/.vite will not find the dependency if it was not hoisted + // (that is, if it is under node_modules directory in the package source of the optimized file) + for (const optimizedModule of [ + ...Object.values(optimizedDeps.metadata.optimized), + ...Object.values(optimizedDeps.metadata.discovered) + ]) { + if (optimizedModule.file === importerModule.file) { + importerFile = optimizedModule.src + } } } } @@ -271,7 +274,11 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { // (e.g. vue blocks), inherit importer's version query // do not do this for unknown type imports, otherwise the appended // query can break 3rd party plugin's extension checks. - if ((isRelative || isSelfImport) && !/[\?&]import=?\b/.test(url)) { + if ( + (isRelative || isSelfImport) && + !/[\?&]import=?\b/.test(url) && + !url.match(DEP_VERSION_RE) + ) { const versionMatch = importer.match(DEP_VERSION_RE) if (versionMatch) { url = injectQuery(url, versionMatch[1]) @@ -446,7 +453,7 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { const file = cleanUrl(resolvedId) // Remove ?v={hash} const needsInterop = await optimizedDepNeedsInterop( - server._optimizeDepsMetadata!, + server._optimizedDeps!.metadata, file ) diff --git a/packages/vite/src/node/plugins/optimizedDeps.ts b/packages/vite/src/node/plugins/optimizedDeps.ts index 8fbdca8d08905f..d92b3467afe700 100644 --- a/packages/vite/src/node/plugins/optimizedDeps.ts +++ b/packages/vite/src/node/plugins/optimizedDeps.ts @@ -25,7 +25,7 @@ export function optimizedDepsPlugin(): Plugin { async load(id) { if (server && isOptimizedDepFile(id, server.config)) { - const metadata = server?._optimizeDepsMetadata + const metadata = server?._optimizedDeps?.metadata if (metadata) { const file = cleanUrl(id) const versionMatch = id.match(DEP_VERSION_RE) @@ -49,7 +49,7 @@ export function optimizedDepsPlugin(): Plugin { throwProcessingError(id) return } - const newMetadata = server._optimizeDepsMetadata + const newMetadata = server._optimizedDeps?.metadata if (metadata !== newMetadata) { const currentInfo = optimizeDepInfoFromFile(newMetadata!, file) if (info.browserHash !== currentInfo?.browserHash) { diff --git a/packages/vite/src/node/plugins/preAlias.ts b/packages/vite/src/node/plugins/preAlias.ts index 75a0d8e5e6f9dc..dadb16aa4c28a9 100644 --- a/packages/vite/src/node/plugins/preAlias.ts +++ b/packages/vite/src/node/plugins/preAlias.ts @@ -13,9 +13,9 @@ export function preAliasPlugin(): Plugin { configureServer(_server) { server = _server }, - resolveId(id, importer, options) { - if (!options?.ssr && bareImportRE.test(id)) { - return tryOptimizedResolve(id, server, importer) + async resolveId(id, importer, options) { + if (!options?.ssr && bareImportRE.test(id) && !options?.scan) { + return await tryOptimizedResolve(id, server, importer) } } } diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index 683c82836ff39d..3ca92d30eb2f69 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -7,7 +7,8 @@ import { SPECIAL_QUERY_RE, DEFAULT_EXTENSIONS, DEFAULT_MAIN_FIELDS, - OPTIMIZABLE_ENTRY_RE + OPTIMIZABLE_ENTRY_RE, + DEP_VERSION_RE } from '../constants' import { isBuiltin, @@ -29,7 +30,11 @@ import { isPossibleTsOutput, getPotentialTsSrcPaths } from '../utils' -import { createIsOptimizedDepUrl } from '../optimizer' +import { + createIsOptimizedDepUrl, + isOptimizedDepFile, + optimizeDepInfoFromFile +} from '../optimizer' import type { OptimizedDepInfo } from '../optimizer' import type { ViteDevServer, SSROptions } from '..' import type { PartialResolvedId } from 'rollup' @@ -78,6 +83,8 @@ export interface InternalResolveOptions extends ResolveOptions { // should also try import from `.ts/tsx/mts/cts` source file as fallback. isFromTsImporter?: boolean tryEsmOnly?: boolean + // True when resolving during the scan phase to discover dependencies + scan?: boolean } export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { @@ -101,7 +108,7 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { isOptimizedDepUrl = createIsOptimizedDepUrl(server.config) }, - resolveId(id, importer, resolveOpts) { + async resolveId(id, importer, resolveOpts) { const ssr = resolveOpts?.ssr === true if (id.startsWith(browserExternalId)) { return id @@ -122,7 +129,8 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { isRequire, ...baseOptions, - isFromTsImporter: isTsRequest(importer ?? '') + isFromTsImporter: isTsRequest(importer ?? ''), + scan: resolveOpts?.scan ?? baseOptions.scan } let res: string | PartialResolvedId | undefined @@ -131,9 +139,10 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { // tryFileResolve or /fs/ resolution but these files may not yet // exists if we are in the middle of a deps re-processing if (asSrc && isOptimizedDepUrl?.(id)) { - return id.startsWith(FS_PREFIX) + const optimizedPath = id.startsWith(FS_PREFIX) ? fsPathFromId(id) : normalizePath(ensureVolumeInPath(path.resolve(root, id.slice(1)))) + return optimizedPath } // explicit fs paths that starts with /@fs/* @@ -163,6 +172,22 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { // handle browser field mapping for relative imports const normalizedFsPath = normalizePath(fsPath) + + if (server && isOptimizedDepFile(normalizedFsPath, server!.config)) { + // Optimized files could not yet exist in disk, resolve to the full path + // Inject the current browserHash version if the path doesn't have one + if (!normalizedFsPath.match(DEP_VERSION_RE)) { + const browserHash = optimizeDepInfoFromFile( + server._optimizedDeps!.metadata!, + normalizedFsPath + )?.browserHash + if (browserHash) { + return injectQuery(normalizedFsPath, `v=${browserHash}`) + } + } + return normalizedFsPath + } + const pathFromBasedir = normalizedFsPath.slice(basedir.length) if (pathFromBasedir.startsWith('/node_modules/')) { // normalize direct imports from node_modules to bare imports, so the @@ -231,7 +256,8 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { asSrc && server && !ssr && - (res = tryOptimizedResolve(id, server, importer)) + !options.scan && + (res = await tryOptimizedResolve(id, server, importer)) ) { return res } @@ -585,7 +611,7 @@ export function tryNodeResolve( if ( !resolved.includes('node_modules') || // linked !server || // build - !server._registerMissingImport // initial esbuild scan phase + options.scan // initial esbuild scan phase ) { return { id: resolved } } @@ -606,14 +632,18 @@ export function tryNodeResolve( // otherwise we may introduce duplicated modules for externalized files // from pre-bundled deps. - const versionHash = server._optimizeDepsMetadata?.browserHash + const versionHash = server._optimizedDeps!.metadata.browserHash if (versionHash && isJsType) { resolved = injectQuery(resolved, `v=${versionHash}`) } } else { // this is a missing import, queue optimize-deps re-run and // get a resolved its optimized info - const optimizedInfo = server._registerMissingImport!(id, resolved, ssr) + const optimizedInfo = server._optimizedDeps!.registerMissingImport( + id, + resolved, + ssr + ) resolved = getOptimizedUrl(optimizedInfo) } return { id: resolved! } @@ -623,14 +653,18 @@ export function tryNodeResolve( const getOptimizedUrl = (optimizedData: OptimizedDepInfo) => `${optimizedData.file}?v=${optimizedData.browserHash}` -export function tryOptimizedResolve( +export async function tryOptimizedResolve( id: string, server: ViteDevServer, importer?: string -): string | undefined { - const depData = server._optimizeDepsMetadata +): Promise { + const optimizedDeps = server._optimizedDeps + + if (!optimizedDeps) return - if (!depData) return + await optimizedDeps.scanProcessing + + const depData = optimizedDeps.metadata // check if id has been optimized const isOptimized = depData.optimized[id] @@ -638,12 +672,25 @@ export function tryOptimizedResolve( return getOptimizedUrl(isOptimized) } + const isChunk = depData.chunks[id] + if (isChunk) { + return getOptimizedUrl(isChunk) + } + + const isDiscovered = depData.discovered[id] + if (isDiscovered) { + return getOptimizedUrl(isDiscovered) + } + if (!importer) return // further check if id is imported by nested dependency let resolvedSrc: string | undefined - for (const [pkgPath, optimizedData] of Object.entries(depData.optimized)) { + for (const [pkgPath, optimizedData] of [ + ...Object.entries(depData.optimized), + ...Object.entries(depData.discovered) + ]) { // check for scenarios, e.g. // pkgPath => "my-lib > foo" // id => "foo" diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index d56a6a705d5416..c88db363dfa7f2 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -44,8 +44,6 @@ import { transformRequest } from './transformRequest' import type { ESBuildTransformResult } from '../plugins/esbuild' import { transformWithEsbuild } from '../plugins/esbuild' import type { TransformOptions as EsbuildTransformOptions } from 'esbuild' -import type { DepOptimizationMetadata, OptimizedDepInfo } from '../optimizer' -import { createOptimizeDepsRun } from '../optimizer' import { ssrLoadModule } from '../ssr/ssrModuleLoader' import { resolveSSRExternal } from '../ssr/ssrExternal' import { @@ -53,7 +51,8 @@ import { ssrRewriteStacktrace } from '../ssr/ssrStacktrace' import { ssrTransform } from '../ssr/ssrTransform' -import { createMissingImporterRegisterFn } from '../optimizer/registerMissing' +import { createOptimizedDeps } from '../optimizer/registerMissing' +import type { OptimizedDeps } from '../optimizer' import { resolveHostname } from '../utils' import { searchForWorkspaceRoot } from './searchRoot' import { CLIENT_DIR } from '../constants' @@ -257,7 +256,7 @@ export interface ViteDevServer { /** * @internal */ - _optimizeDepsMetadata: DepOptimizationMetadata | null + _optimizedDeps: OptimizedDeps | null /** * Deps that are externalized * @internal @@ -284,16 +283,6 @@ export interface ViteDevServer { * @internal */ _forceOptimizeOnRestart: boolean - /** - * @internal - */ - _registerMissingImport: - | (( - id: string, - resolved: string, - ssr: boolean | undefined - ) => OptimizedDepInfo) - | null /** * @internal */ @@ -373,9 +362,14 @@ export async function createServer( transformIndexHtml: null!, // to be immediately set async ssrLoadModule(url, opts?: { fixStacktrace?: boolean }) { let configFileDependencies: string[] = [] - const metadata = server._optimizeDepsMetadata - if (metadata) { - configFileDependencies = Object.keys(metadata.optimized) + const optimizedDeps = server._optimizedDeps + if (optimizedDeps) { + await optimizedDeps.scanProcessing + const { metadata } = optimizedDeps + configFileDependencies = [ + ...Object.keys(metadata.optimized), + ...Object.keys(metadata.discovered) + ] } server._ssrExternals ||= resolveSSRExternal( @@ -434,12 +428,11 @@ export async function createServer( return server._restartPromise }, - _optimizeDepsMetadata: null, + _optimizedDeps: null, _ssrExternals: null, _globImporters: Object.create(null), _restartPromise: null, _forceOptimizeOnRestart: false, - _registerMissingImport: null, _pendingRequests: new Map() } @@ -582,27 +575,7 @@ export async function createServer( middlewares.use(errorMiddleware(server, !!middlewareMode)) const runOptimize = async () => { - const optimizeDeps = await createOptimizeDepsRun( - config, - config.server.force - ) - - // Don't await for the optimization to finish, we can start the - // server right away here - server._optimizeDepsMetadata = optimizeDeps.metadata - - // Run deps optimization in parallel - const initialProcessingPromise = optimizeDeps - .run() - .then((result) => result.commit()) - - // While running the first optimizeDeps, _registerMissingImport is null - // so the resolve plugin resolves straight to node_modules during the - // deps discovery scan phase - server._registerMissingImport = createMissingImporterRegisterFn( - server, - initialProcessingPromise - ) + server._optimizedDeps = await createOptimizedDeps(server) } if (!middlewareMode && httpServer) { diff --git a/packages/vite/src/node/server/pluginContainer.ts b/packages/vite/src/node/server/pluginContainer.ts index 2f9c171de5d2e5..c32dd7db89127a 100644 --- a/packages/vite/src/node/server/pluginContainer.ts +++ b/packages/vite/src/node/server/pluginContainer.ts @@ -90,6 +90,7 @@ export interface PluginContainer { options?: { skip?: Set ssr?: boolean + scan?: boolean } ): Promise transform( @@ -212,6 +213,7 @@ export async function createPluginContainer( class Context implements PluginContext { meta = minimalContext.meta ssr = false + _scan = false _activePlugin: Plugin | null _activeId: string | null = null _activeCode: string | null = null @@ -241,7 +243,11 @@ export async function createPluginContainer( skip = new Set(this._resolveSkips) skip.add(this._activePlugin) } - let out = await container.resolveId(id, importer, { skip, ssr: this.ssr }) + let out = await container.resolveId(id, importer, { + skip, + ssr: this.ssr, + scan: this._scan + }) if (typeof out === 'string') out = { id: out } return out as ResolvedId | null } @@ -487,8 +493,10 @@ export async function createPluginContainer( async resolveId(rawId, importer = join(root, 'index.html'), options) { const skip = options?.skip const ssr = options?.ssr + const scan = !!options?.scan const ctx = new Context() ctx.ssr = !!ssr + ctx._scan = scan ctx._resolveSkips = skip const resolveStart = isDebug ? performance.now() : 0 @@ -505,7 +513,7 @@ export async function createPluginContainer( ctx as any, rawId, importer, - { ssr } + { ssr, scan } ) if (!result) continue From f15c7796626a017356eecf49c1b04d85eea75221 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sat, 19 Mar 2022 06:33:54 +0100 Subject: [PATCH 02/18] fix: calling resolve before listening to the server --- packages/vite/src/node/optimizer/registerMissing.ts | 4 ++-- packages/vite/src/node/plugins/importAnalysis.ts | 1 + packages/vite/src/node/plugins/resolve.ts | 6 +++++- packages/vite/src/node/server/index.ts | 10 +++------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index ebe7f97181444f..7e17a6ede7bab7 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -25,10 +25,10 @@ import { resolveSSRExternal } from '../ssr/ssrExternal' */ const debounceMs = 100 -export async function createOptimizedDeps( +export function createOptimizedDeps( server: ViteDevServer, asCommand = false -): Promise { +): OptimizedDeps { const { config } = server const { logger } = config diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts index 1dc213b2b42f61..495f8f0580c39e 100644 --- a/packages/vite/src/node/plugins/importAnalysis.ts +++ b/packages/vite/src/node/plugins/importAnalysis.ts @@ -442,6 +442,7 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { importRewrites.push(async () => { let rewriteDone = false if ( + server?._optimizedDeps && isOptimizedDepFile(resolvedId, config) && !resolvedId.match(optimizedDepChunkRE) ) { diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index 3ca92d30eb2f69..bcb1dfed389df8 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -173,7 +173,10 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { const normalizedFsPath = normalizePath(fsPath) - if (server && isOptimizedDepFile(normalizedFsPath, server!.config)) { + if ( + server?._optimizedDeps && + isOptimizedDepFile(normalizedFsPath, server!.config) + ) { // Optimized files could not yet exist in disk, resolve to the full path // Inject the current browserHash version if the path doesn't have one if (!normalizedFsPath.match(DEP_VERSION_RE)) { @@ -611,6 +614,7 @@ export function tryNodeResolve( if ( !resolved.includes('node_modules') || // linked !server || // build + !server._optimizedDeps || // resolving before listening to the server options.scan // initial esbuild scan phase ) { return { id: resolved } diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index c88db363dfa7f2..03d4f399fcb9e1 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -574,19 +574,15 @@ export async function createServer( // error handler middlewares.use(errorMiddleware(server, !!middlewareMode)) - const runOptimize = async () => { - server._optimizedDeps = await createOptimizedDeps(server) - } - if (!middlewareMode && httpServer) { let isOptimized = false - // overwrite listen to run optimizer before server start + // overwrite listen to init optimizer before server start const listen = httpServer.listen.bind(httpServer) httpServer.listen = (async (port: number, ...args: any[]) => { if (!isOptimized) { try { await container.buildStart({}) - await runOptimize() + server._optimizedDeps = createOptimizedDeps(server) isOptimized = true } catch (e) { httpServer.emit('error', e) @@ -597,7 +593,7 @@ export async function createServer( }) as any } else { await container.buildStart({}) - await runOptimize() + server._optimizedDeps = createOptimizedDeps(server) } return server From 06f1a43d8a27b020b5d337128e369bede9efa566 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sat, 19 Mar 2022 10:07:15 +0100 Subject: [PATCH 03/18] refactor: findOptimizedDepInfo --- packages/vite/src/node/optimizer/index.ts | 51 ++++++++++--- .../vite/src/node/plugins/importAnalysis.ts | 20 ++--- .../vite/src/node/plugins/optimizedDeps.ts | 6 +- packages/vite/src/node/plugins/resolve.ts | 76 ++++++++----------- 4 files changed, 87 insertions(+), 66 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 0f8b15e7433d02..63abacb0dfe3ab 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -361,7 +361,7 @@ export async function createOptimizeDepsRun( // We prebundle dependencies with esbuild and cache them, but there is no need // to wait here. Code that needs to access the cached deps needs to await - // the optimizeDepInfo.processing promise for each dep + // the optimizedDepInfo.processing promise for each dep const qualifiedIds = Object.keys(deps) @@ -523,7 +523,12 @@ export async function createOptimizeDepsRun( .relative(processingCacheDirOutputPath, o) .replace(jsExtensionRE, '') const file = getOptimizedDepPath(id, config) - if (!findFileInfo(metadata.optimized, file)) { + if ( + !findOptimizedDepInfoInRecord( + metadata.optimized, + (depInfo) => depInfo.file === file + ) + ) { metadata.chunks[id] = { file, src: '', @@ -849,24 +854,48 @@ function getDepHash(config: ResolvedConfig): string { return createHash('sha256').update(content).digest('hex').substring(0, 8) } -export function optimizeDepInfoFromFile( +export function optimizedDepInfoFromId( metadata: DepOptimizationMetadata, - file: string + id: string ): OptimizedDepInfo | undefined { return ( - findFileInfo(metadata.optimized, file) || - findFileInfo(metadata.discovered, file) || - findFileInfo(metadata.chunks, file) + metadata.optimized[id] || metadata.discovered[id] || metadata.chunks[id] + ) +} + +export function optimizedDepInfoFromFile( + metadata: DepOptimizationMetadata, + file: string, + includeChunks: boolean = true +): OptimizedDepInfo | undefined { + return findOptimizedDepInfo( + metadata, + (depInfo) => depInfo.file === file, + includeChunks ) } -function findFileInfo( +export function findOptimizedDepInfo( + metadata: DepOptimizationMetadata, + callbackFn: (depInfo: OptimizedDepInfo, id: string) => any, + includeChunks: boolean = true +): OptimizedDepInfo | undefined { + return ( + findOptimizedDepInfoInRecord(metadata.optimized, callbackFn) || + findOptimizedDepInfoInRecord(metadata.discovered, callbackFn) || + (includeChunks + ? findOptimizedDepInfoInRecord(metadata.chunks, callbackFn) + : undefined) + ) +} + +function findOptimizedDepInfoInRecord( dependenciesInfo: Record, - file: string + callbackFn: (depInfo: OptimizedDepInfo, id: string) => any ): OptimizedDepInfo | undefined { for (const o of Object.keys(dependenciesInfo)) { const info = dependenciesInfo[o] - if (info.file === file) { + if (callbackFn(info, o)) { return info } } @@ -876,7 +905,7 @@ export async function optimizedDepNeedsInterop( metadata: DepOptimizationMetadata, file: string ): Promise { - const depInfo = optimizeDepInfoFromFile(metadata, file) + const depInfo = optimizedDepInfoFromFile(metadata, file) if (!depInfo) return undefined diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts index e31b76ac1650ae..23b56b4b07160d 100644 --- a/packages/vite/src/node/plugins/importAnalysis.ts +++ b/packages/vite/src/node/plugins/importAnalysis.ts @@ -50,7 +50,8 @@ import { transformRequest } from '../server/transformRequest' import { isOptimizedDepFile, getDepsCacheDir, - optimizedDepNeedsInterop + optimizedDepNeedsInterop, + findOptimizedDepInfo } from '../optimizer' const isDebug = !!process.env.DEBUG @@ -206,14 +207,15 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { // the dependency needs to be resolved starting from the original source location of the optimized file // because starting from node_modules/.vite will not find the dependency if it was not hoisted // (that is, if it is under node_modules directory in the package source of the optimized file) - for (const optimizedModule of [ - ...Object.values(optimizedDeps.metadata.optimized), - ...Object.values(optimizedDeps.metadata.discovered) - ]) { - if (optimizedModule.file === importerModule.file) { - importerFile = optimizedModule.src - } - } + findOptimizedDepInfo( + optimizedDeps.metadata, + (optimizedModule) => { + if (optimizedModule.file === importerModule.file) { + importerFile = optimizedModule.src + } + }, + false + ) // don't search chunks } } diff --git a/packages/vite/src/node/plugins/optimizedDeps.ts b/packages/vite/src/node/plugins/optimizedDeps.ts index d92b3467afe700..adab1bd9756251 100644 --- a/packages/vite/src/node/plugins/optimizedDeps.ts +++ b/packages/vite/src/node/plugins/optimizedDeps.ts @@ -3,7 +3,7 @@ import type { Plugin } from '../plugin' import colors from 'picocolors' import { DEP_VERSION_RE } from '../constants' import { cleanUrl, createDebugger } from '../utils' -import { isOptimizedDepFile, optimizeDepInfoFromFile } from '../optimizer' +import { isOptimizedDepFile, optimizedDepInfoFromFile } from '../optimizer' import type { ViteDevServer } from '..' export const ERR_OPTIMIZE_DEPS_PROCESSING_ERROR = @@ -34,7 +34,7 @@ export function optimizedDepsPlugin(): Plugin { : undefined // Search in both the currently optimized and newly discovered deps - const info = optimizeDepInfoFromFile(metadata, file) + const info = optimizedDepInfoFromFile(metadata, file) if (info) { if (browserHash && info.browserHash !== browserHash) { throwOutdatedRequest(id) @@ -51,7 +51,7 @@ export function optimizedDepsPlugin(): Plugin { } const newMetadata = server._optimizedDeps?.metadata if (metadata !== newMetadata) { - const currentInfo = optimizeDepInfoFromFile(newMetadata!, file) + const currentInfo = optimizedDepInfoFromFile(newMetadata!, file) if (info.browserHash !== currentInfo?.browserHash) { throwOutdatedRequest(id) } diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index bcb1dfed389df8..0533b8c5dc180f 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -32,8 +32,10 @@ import { } from '../utils' import { createIsOptimizedDepUrl, + findOptimizedDepInfo, isOptimizedDepFile, - optimizeDepInfoFromFile + optimizedDepInfoFromFile, + optimizedDepInfoFromId } from '../optimizer' import type { OptimizedDepInfo } from '../optimizer' import type { ViteDevServer, SSROptions } from '..' @@ -180,7 +182,7 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin { // Optimized files could not yet exist in disk, resolve to the full path // Inject the current browserHash version if the path doesn't have one if (!normalizedFsPath.match(DEP_VERSION_RE)) { - const browserHash = optimizeDepInfoFromFile( + const browserHash = optimizedDepInfoFromFile( server._optimizedDeps!.metadata!, normalizedFsPath )?.browserHash @@ -670,52 +672,40 @@ export async function tryOptimizedResolve( const depData = optimizedDeps.metadata - // check if id has been optimized - const isOptimized = depData.optimized[id] - if (isOptimized) { - return getOptimizedUrl(isOptimized) - } - - const isChunk = depData.chunks[id] - if (isChunk) { - return getOptimizedUrl(isChunk) - } - - const isDiscovered = depData.discovered[id] - if (isDiscovered) { - return getOptimizedUrl(isDiscovered) + const depInfo = optimizedDepInfoFromId(depData, id) + if (depInfo) { + return getOptimizedUrl(depInfo) } if (!importer) return - // further check if id is imported by nested dependency - let resolvedSrc: string | undefined - - for (const [pkgPath, optimizedData] of [ - ...Object.entries(depData.optimized), - ...Object.entries(depData.discovered) - ]) { - // check for scenarios, e.g. - // pkgPath => "my-lib > foo" - // id => "foo" - // this narrows the need to do a full resolve - if (!pkgPath.endsWith(id)) continue - - // lazily initialize resolvedSrc - if (resolvedSrc == null) { - try { - // this may throw errors if unable to resolve, e.g. aliased id - resolvedSrc = normalizePath(resolveFrom(id, path.dirname(importer))) - } catch { - // this is best-effort only so swallow errors - break - } - } + try { + // further check if id is imported by nested dependency + let resolvedSrc: string | undefined + + const nestedDepInfo = findOptimizedDepInfo( + depData, + (optimizedData, pkgPath) => { + // check for scenarios, e.g. + // pkgPath => "my-lib > foo" + // id => "foo" + // this narrows the need to do a full resolve + if (!pkgPath.endsWith(id)) return false + + // lazily initialize resolvedSrc + if (resolvedSrc == null) { + // this may throw errors if unable to resolve, e.g. aliased id + resolvedSrc = normalizePath(resolveFrom(id, path.dirname(importer))) + } - // match by src to correctly identify if id belongs to nested dependency - if (optimizedData.src === resolvedSrc) { - return getOptimizedUrl(optimizedData) - } + // match by src to correctly identify if id belongs to nested dependency + return optimizedData.src === resolvedSrc + }, + false // don't search chunks + ) + if (nestedDepInfo) return getOptimizedUrl(nestedDepInfo) + } catch { + // this is best-effort only so swallow errors } } From 3e8348ed1b3753fd67f4dc814eb5d487722cc05e Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sat, 19 Mar 2022 22:24:26 +0100 Subject: [PATCH 04/18] perf: optimize depInfo iteration --- packages/vite/src/node/optimizer/index.ts | 51 ++++++++-------- .../src/node/optimizer/registerMissing.ts | 15 +++-- .../vite/src/node/plugins/importAnalysis.ts | 18 +++--- packages/vite/src/node/plugins/resolve.ts | 58 +++++++++---------- 4 files changed, 68 insertions(+), 74 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 63abacb0dfe3ab..53fd4f24842238 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -130,6 +130,7 @@ export interface DepOptimizationProcessing { } export interface OptimizedDepInfo { + id: string file: string src: string needsInterop?: boolean @@ -166,6 +167,10 @@ export interface DepOptimizationMetadata { * Metadata for each newly discovered dependency after processing */ discovered: Record + /** + * OptimizedDepInfo list + */ + depInfoList: OptimizedDepInfo[] } /** @@ -203,7 +208,8 @@ export function createOptimizedDepsMetadata(config: ResolvedConfig) { browserHash: mainHash, optimized: {}, chunks: {}, - discovered: {} + discovered: {}, + depInfoList: [] } } @@ -281,6 +287,7 @@ export async function discoverProjectDependencies( for (const id in deps) { const entry = deps[id] discovered[id] = { + id, file: getOptimizedDepPath(id, config), src: entry, browserHash: browserHash @@ -356,7 +363,8 @@ export async function createOptimizeDepsRun( browserHash: currentData?.browserHash || newBrowserHash, optimized: depsInfo, chunks: {}, - discovered: {} + discovered: {}, + depInfoList: Object.values(depsInfo) } // We prebundle dependencies with esbuild and cache them, but there is no need @@ -530,6 +538,7 @@ export async function createOptimizeDepsRun( ) ) { metadata.chunks[id] = { + id, file, src: '', needsInterop: false, @@ -702,15 +711,20 @@ function parseOptimizedDepsMetadata( return value }) const { browserHash } = metadata - for (const o of Object.keys(metadata.optimized)) { - const depInfo = metadata.optimized[o] + metadata.depInfoList = [] + for (const id of Object.keys(metadata.optimized)) { + const depInfo = metadata.optimized[id] + depInfo.id = id depInfo.browserHash = browserHash + metadata.depInfoList.push(depInfo) } metadata.chunks ||= {} // Support missing chunks for back compat - for (const o of Object.keys(metadata.chunks)) { - const depInfo = metadata.chunks[o] + for (const id of Object.keys(metadata.chunks)) { + const depInfo = metadata.chunks[id] + depInfo.id = id depInfo.src = '' depInfo.browserHash = browserHash + metadata.depInfoList.push(depInfo) } metadata.discovered = {} return metadata @@ -729,7 +743,7 @@ function stringifyOptimizedDepsMetadata( return JSON.stringify( metadata, (key: string, value: any) => { - if (key === 'discovered' || key === 'processing') { + if (key === 'discovered' || key === 'processing' || key === 'id') { return } if (key === 'file' || key === 'src') { @@ -865,28 +879,9 @@ export function optimizedDepInfoFromId( export function optimizedDepInfoFromFile( metadata: DepOptimizationMetadata, - file: string, - includeChunks: boolean = true -): OptimizedDepInfo | undefined { - return findOptimizedDepInfo( - metadata, - (depInfo) => depInfo.file === file, - includeChunks - ) -} - -export function findOptimizedDepInfo( - metadata: DepOptimizationMetadata, - callbackFn: (depInfo: OptimizedDepInfo, id: string) => any, - includeChunks: boolean = true + file: string ): OptimizedDepInfo | undefined { - return ( - findOptimizedDepInfoInRecord(metadata.optimized, callbackFn) || - findOptimizedDepInfoInRecord(metadata.discovered, callbackFn) || - (includeChunks - ? findOptimizedDepInfoInRecord(metadata.chunks, callbackFn) - : undefined) - ) + return metadata.depInfoList.find((depInfo) => depInfo.file === file) } function findOptimizedDepInfoInRecord( diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 3c11723197c763..686a7d50238f9d 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -34,10 +34,10 @@ export function createOptimizedDeps( const cachedMetadata = loadCachedDepOptimizationMetadata(config) - const optimizedDeps = { + const optimizedDeps: OptimizedDeps = { metadata: cachedMetadata || createOptimizedDepsMetadata(config), registerMissingImport - } as OptimizedDeps + } let handle: NodeJS.Timeout | undefined let newDepsDiscovered = false @@ -86,6 +86,7 @@ export function createOptimizedDeps( }) metadata.discovered = discovered + metadata.depInfoList = Object.values(discovered) scanPhaseProcessing.resolve() optimizedDeps.scanProcessing = undefined @@ -172,9 +173,11 @@ export function createOptimizedDeps( // While optimizeDeps is running, new missing deps may be discovered, // in which case they will keep being added to metadata.discovered - for (const o of Object.keys(metadata.discovered)) { - if (!newData.optimized[o]) { - newData.discovered[o] = metadata.discovered[o] + for (const id of Object.keys(metadata.discovered)) { + if (!newData.optimized[id]) { + const depInfo = metadata.discovered[id] + newData.discovered[id] = depInfo + newData.depInfoList.push(depInfo) } } @@ -313,6 +316,7 @@ export function createOptimizedDeps( } newDepsDiscovered = true missing = metadata.discovered[id] = { + id, file: getOptimizedDepPath(id, server.config), src: resolved, // Assing a browserHash to this missing dependency that is unique to @@ -328,6 +332,7 @@ export function createOptimizedDeps( // promise to be resolved processing: depOptimizationProcessing.promise } + metadata.depInfoList.push(missing) // Debounced rerun, let other missing dependencies be discovered before // the running next optimizeDeps diff --git a/packages/vite/src/node/plugins/importAnalysis.ts b/packages/vite/src/node/plugins/importAnalysis.ts index 23b56b4b07160d..df3b8527ba76ca 100644 --- a/packages/vite/src/node/plugins/importAnalysis.ts +++ b/packages/vite/src/node/plugins/importAnalysis.ts @@ -50,8 +50,7 @@ import { transformRequest } from '../server/transformRequest' import { isOptimizedDepFile, getDepsCacheDir, - optimizedDepNeedsInterop, - findOptimizedDepInfo + optimizedDepNeedsInterop } from '../optimizer' const isDebug = !!process.env.DEBUG @@ -207,15 +206,12 @@ export function importAnalysisPlugin(config: ResolvedConfig): Plugin { // the dependency needs to be resolved starting from the original source location of the optimized file // because starting from node_modules/.vite will not find the dependency if it was not hoisted // (that is, if it is under node_modules directory in the package source of the optimized file) - findOptimizedDepInfo( - optimizedDeps.metadata, - (optimizedModule) => { - if (optimizedModule.file === importerModule.file) { - importerFile = optimizedModule.src - } - }, - false - ) // don't search chunks + for (const optimizedModule of optimizedDeps.metadata.depInfoList) { + if (!optimizedModule.src) continue // Ignore chunks + if (optimizedModule.file === importerModule.file) { + importerFile = optimizedModule.src + } + } } } diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index 0533b8c5dc180f..828deb00f9c837 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -32,7 +32,6 @@ import { } from '../utils' import { createIsOptimizedDepUrl, - findOptimizedDepInfo, isOptimizedDepFile, optimizedDepInfoFromFile, optimizedDepInfoFromId @@ -670,42 +669,41 @@ export async function tryOptimizedResolve( await optimizedDeps.scanProcessing - const depData = optimizedDeps.metadata - - const depInfo = optimizedDepInfoFromId(depData, id) + const depInfo = optimizedDepInfoFromId(optimizedDeps.metadata, id) if (depInfo) { return getOptimizedUrl(depInfo) } if (!importer) return - try { - // further check if id is imported by nested dependency - let resolvedSrc: string | undefined - - const nestedDepInfo = findOptimizedDepInfo( - depData, - (optimizedData, pkgPath) => { - // check for scenarios, e.g. - // pkgPath => "my-lib > foo" - // id => "foo" - // this narrows the need to do a full resolve - if (!pkgPath.endsWith(id)) return false - - // lazily initialize resolvedSrc - if (resolvedSrc == null) { - // this may throw errors if unable to resolve, e.g. aliased id - resolvedSrc = normalizePath(resolveFrom(id, path.dirname(importer))) - } + // further check if id is imported by nested dependency + let resolvedSrc: string | undefined + + for (const optimizedData of optimizedDeps.metadata.depInfoList) { + if (!optimizedData.src) continue // Ignore chunks + + const pkgPath = optimizedData.id + // check for scenarios, e.g. + // pkgPath => "my-lib > foo" + // id => "foo" + // this narrows the need to do a full resolve + if (!pkgPath.endsWith(id)) continue + + // lazily initialize resolvedSrc + if (resolvedSrc == null) { + try { + // this may throw errors if unable to resolve, e.g. aliased id + resolvedSrc = normalizePath(resolveFrom(id, path.dirname(importer))) + } catch { + // this is best-effort only so swallow errors + break + } + } - // match by src to correctly identify if id belongs to nested dependency - return optimizedData.src === resolvedSrc - }, - false // don't search chunks - ) - if (nestedDepInfo) return getOptimizedUrl(nestedDepInfo) - } catch { - // this is best-effort only so swallow errors + // match by src to correctly identify if id belongs to nested dependency + if (optimizedData.src === resolvedSrc) { + return getOptimizedUrl(optimizedData) + } } } From ace8a7799931af30274fe8531f01d19f01046a16 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sun, 20 Mar 2022 09:01:27 +0100 Subject: [PATCH 05/18] chore: addOptimizedDepInfo --- packages/vite/src/node/optimizer/index.ts | 56 +++++++++++-------- .../src/node/optimizer/registerMissing.ts | 20 +++---- 2 files changed, 41 insertions(+), 35 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 53fd4f24842238..661d5b2f3ddce0 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -201,7 +201,9 @@ export async function optimizeDeps( return result.metadata } -export function createOptimizedDepsMetadata(config: ResolvedConfig) { +export function createOptimizedDepsMetadata( + config: ResolvedConfig +): DepOptimizationMetadata { const mainHash = getDepHash(config) return { hash: mainHash, @@ -213,6 +215,16 @@ export function createOptimizedDepsMetadata(config: ResolvedConfig) { } } +export function addOptimizedDepInfo( + metadata: DepOptimizationMetadata, + type: 'optimized' | 'discovered' | 'chunks', + depInfo: OptimizedDepInfo +): OptimizedDepInfo { + metadata[type][depInfo.id] = depInfo + metadata.depInfoList.push(depInfo) + return depInfo +} + /** * Creates the initial dep optimization metadata, loading it from the deps cache * if it exists and pre-bundling isn't forced @@ -356,16 +368,11 @@ export async function createOptimizeDepsRun( const newBrowserHash = getOptimizedBrowserHash(mainHash, deps) - const metadata: DepOptimizationMetadata = { - hash: mainHash, - // For reruns keep current global browser hash and newDeps individual hashes until we know - // if files are stable so we can avoid a full page reload - browserHash: currentData?.browserHash || newBrowserHash, - optimized: depsInfo, - chunks: {}, - discovered: {}, - depInfoList: Object.values(depsInfo) - } + const metadata = createOptimizedDepsMetadata(config) + + // For reruns keep current global browser hash and newDeps individual hashes until we know + // if files are stable so we can avoid a full page reload + metadata.browserHash = currentData?.browserHash || newBrowserHash // We prebundle dependencies with esbuild and cache them, but there is no need // to wait here. Code that needs to access the cached deps needs to await @@ -487,24 +494,25 @@ export async function createOptimizeDepsRun( ) for (const id in deps) { - const optimizedInfo = metadata.optimized[id] - optimizedInfo.needsInterop = needsInterop( - id, - idToExports[id], - meta.outputs, - processingCacheDirOutputPath - ) const output = meta.outputs[ path.relative(process.cwd(), getProcessingDepPath(id, config)) ] - if (output) { + + addOptimizedDepInfo(metadata, 'optimized', { + ...depsInfo[id], + needsInterop: needsInterop( + id, + idToExports[id], + meta.outputs, + processingCacheDirOutputPath + ), // We only need to hash the output.imports in to check for stability, but adding the hash // and file path gives us a unique hash that may be useful for other things in the future - optimizedInfo.fileHash = getHash( - metadata.hash + optimizedInfo.file + JSON.stringify(output.imports) + fileHash: getHash( + metadata.hash + depsInfo[id].file + JSON.stringify(output.imports) ) - } + }) } // This only runs when missing deps are processed. Previous optimized deps are stable if @@ -537,7 +545,7 @@ export async function createOptimizeDepsRun( (depInfo) => depInfo.file === file ) ) { - metadata.chunks[id] = { + addOptimizedDepInfo(metadata, 'chunks', { id, file, src: '', @@ -545,7 +553,7 @@ export async function createOptimizeDepsRun( browserHash: (!alteredFiles && currentData?.chunks[id]?.browserHash) || newBrowserHash - } + }) } } } diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 686a7d50238f9d..4802e64a072ccc 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -7,6 +7,7 @@ import { newDepOptimizationProcessing, loadCachedDepOptimizationMetadata, createOptimizedDepsMetadata, + addOptimizedDepInfo, discoverProjectDependencies, depsLogString } from '.' @@ -74,8 +75,11 @@ export function createOptimizedDeps( const discovered = await discoverProjectDependencies(config) // Respect the scan phase discover order to improve reproducibility - for (const dep of Object.keys(discovered)) { - discovered[dep].processing = depOptimizationProcessing.promise + for (const depInfo of Object.values(discovered)) { + addOptimizedDepInfo(metadata, 'discovered', { + ...depInfo, + processing: depOptimizationProcessing.promise + }) } // This is auto run on server start - let the user know that we are @@ -85,9 +89,6 @@ export function createOptimizedDeps( timestamp: true }) - metadata.discovered = discovered - metadata.depInfoList = Object.values(discovered) - scanPhaseProcessing.resolve() optimizedDeps.scanProcessing = undefined @@ -175,9 +176,7 @@ export function createOptimizedDeps( // in which case they will keep being added to metadata.discovered for (const id of Object.keys(metadata.discovered)) { if (!newData.optimized[id]) { - const depInfo = metadata.discovered[id] - newData.discovered[id] = depInfo - newData.depInfoList.push(depInfo) + addOptimizedDepInfo(newData, 'discovered', metadata.discovered[id]) } } @@ -315,7 +314,7 @@ export function createOptimizedDeps( return missing } newDepsDiscovered = true - missing = metadata.discovered[id] = { + missing = addOptimizedDepInfo(metadata, 'discovered', { id, file: getOptimizedDepPath(id, server.config), src: resolved, @@ -331,8 +330,7 @@ export function createOptimizedDeps( // loading of this pre-bundled dep needs to await for its processing // promise to be resolved processing: depOptimizationProcessing.promise - } - metadata.depInfoList.push(missing) + }) // Debounced rerun, let other missing dependencies be discovered before // the running next optimizeDeps From 09eaa693008ecc89ec1fdef957aa74f1cb8bf518 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sun, 20 Mar 2022 14:32:28 +0100 Subject: [PATCH 06/18] chore: update --- packages/vite/src/node/optimizer/index.ts | 91 +++++-------------- .../src/node/optimizer/registerMissing.ts | 60 ++++++------ packages/vite/src/node/plugins/resolve.ts | 3 +- packages/vite/src/node/server/index.ts | 28 +++--- 4 files changed, 72 insertions(+), 110 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 661d5b2f3ddce0..bb93dd76e837bc 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -36,11 +36,7 @@ export type ExportsData = ReturnType & { export type OptimizedDeps = { metadata: DepOptimizationMetadata scanProcessing?: Promise - registerMissingImport: ( - id: string, - resolved: string, - ssr?: boolean - ) => OptimizedDepInfo + registerMissingImport: (id: string, resolved: string) => OptimizedDepInfo } export interface DepOptimizationOptions { @@ -108,13 +104,6 @@ export interface DepOptimizationOptions { export interface DepOptimizationResult { metadata: DepOptimizationMetadata - /** - * After a re-optimization, the internal bundled chunks may change - * and a full page reload is required if that is the case - * If the files are stable, we can avoid the reload that is expensive - * for large applications - */ - alteredFiles: boolean /** * When doing a re-run, if there are newly discovered dependendencies * the page reload will be delayed until the next rerun so we need @@ -132,7 +121,7 @@ export interface DepOptimizationProcessing { export interface OptimizedDepInfo { id: string file: string - src: string + src?: string needsInterop?: boolean browserHash?: string fileHash?: string @@ -194,7 +183,7 @@ export async function optimizeDeps( const depsString = depsLogString(config, Object.keys(depsInfo)) config.logger.info(colors.green(`Optimizing dependencies:\n ${depsString}`)) - const result = await createOptimizeDepsRun(config, depsInfo) + const result = await runOptimizeDeps(config, depsInfo) result.commit() @@ -332,11 +321,9 @@ export function depsLogString( * Internally, Vite uses this function to prepare a optimizeDeps run. When Vite starts, we can get * the metadata and start the server without waiting for the optimizeDeps processing to be completed */ -export async function createOptimizeDepsRun( +export async function runOptimizeDeps( config: ResolvedConfig, - depsInfo: Record, - currentData?: DepOptimizationMetadata, - ssr?: boolean + depsInfo: Record ): Promise { config = { ...config, @@ -364,26 +351,22 @@ export async function createOptimizeDepsRun( JSON.stringify({ type: 'module' }) ) - const deps = depsFromOptimizedDepInfo(depsInfo) - - const newBrowserHash = getOptimizedBrowserHash(mainHash, deps) - const metadata = createOptimizedDepsMetadata(config) - // For reruns keep current global browser hash and newDeps individual hashes until we know - // if files are stable so we can avoid a full page reload - metadata.browserHash = currentData?.browserHash || newBrowserHash + metadata.browserHash = getOptimizedBrowserHash( + mainHash, + depsFromOptimizedDepInfo(depsInfo) + ) // We prebundle dependencies with esbuild and cache them, but there is no need // to wait here. Code that needs to access the cached deps needs to await // the optimizedDepInfo.processing promise for each dep - const qualifiedIds = Object.keys(deps) + const qualifiedIds = Object.keys(depsInfo) if (!qualifiedIds.length) { return { metadata, - alteredFiles: false, commit() { // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` commitProcessingDepsCacheSync() @@ -407,9 +390,9 @@ export async function createOptimizeDepsRun( config.optimizeDeps?.esbuildOptions ?? {} await init - for (const id in deps) { + for (const id in depsInfo) { const flatId = flattenId(id) - const filePath = (flatIdDeps[flatId] = deps[id]) + const filePath = (flatIdDeps[flatId] = depsInfo[id].src!) let exportsData: ExportsData if (config.optimizeDeps.extensions?.some((ext) => filePath.endsWith(ext))) { // For custom supported extensions, build the entry file to transform it into JS, @@ -480,7 +463,7 @@ export async function createOptimizeDepsRun( define, plugins: [ ...plugins, - esbuildDepPlugin(flatIdDeps, flatIdToExports, config, ssr) + esbuildDepPlugin(flatIdDeps, flatIdToExports, config) ], ...esbuildOptions }) @@ -493,7 +476,7 @@ export async function createOptimizeDepsRun( processingCacheDir ) - for (const id in deps) { + for (const id in depsInfo) { const output = meta.outputs[ path.relative(process.cwd(), getProcessingDepPath(id, config)) @@ -511,26 +494,9 @@ export async function createOptimizeDepsRun( // and file path gives us a unique hash that may be useful for other things in the future fileHash: getHash( metadata.hash + depsInfo[id].file + JSON.stringify(output.imports) - ) - }) - } - - // This only runs when missing deps are processed. Previous optimized deps are stable if - // the newly discovered deps don't have common chunks with them. Comparing their fileHash we - // can find out if it is safe to keep the current browser state. If one of the file hashes - // changed, a full page reload is needed - let alteredFiles = false - if (currentData) { - alteredFiles = Object.keys(currentData.optimized).some((dep) => { - const currentInfo = currentData.optimized[dep] - const info = metadata.optimized[dep] - return ( - !info?.fileHash || - !currentInfo?.fileHash || - info?.fileHash !== currentInfo?.fileHash - ) + ), + browserHash: metadata.browserHash }) - debug(`optimized deps have altered files: ${alteredFiles}`) } for (const o of Object.keys(meta.outputs)) { @@ -548,31 +514,17 @@ export async function createOptimizeDepsRun( addOptimizedDepInfo(metadata, 'chunks', { id, file, - src: '', needsInterop: false, - browserHash: - (!alteredFiles && currentData?.chunks[id]?.browserHash) || - newBrowserHash + browserHash: metadata.browserHash }) } } } - if (alteredFiles) { - // Overwrite individual hashes with the new global browserHash, a full page reload is required - // New deps that ended up with a different hash replaced while doing analysis import are going to - // return a not found so the browser doesn't cache them. And will properly get loaded after the reload - for (const id in deps) { - metadata.optimized[id].browserHash = newBrowserHash - } - metadata.browserHash = newBrowserHash - } - debug(`deps bundled in ${(performance.now() - start).toFixed(2)}ms`) return { metadata, - alteredFiles, commit() { // Write metadata file, delete `deps` folder and rename the new `processing` folder to `deps` in sync commitProcessingDepsCacheSync() @@ -647,7 +599,7 @@ export function depsFromOptimizedDepInfo( depsInfo: Record ) { return Object.fromEntries( - Object.entries(depsInfo).map((d) => [d[0], d[1].src]) + Object.entries(depsInfo).map((d) => [d[0], d[1].src!]) ) } @@ -751,7 +703,12 @@ function stringifyOptimizedDepsMetadata( return JSON.stringify( metadata, (key: string, value: any) => { - if (key === 'discovered' || key === 'processing' || key === 'id') { + if ( + key === 'discovered' || + key === 'processing' || + key === 'id' || + key === 'depInfoList' + ) { return } if (key === 'file' || key === 'src') { diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 4802e64a072ccc..5795576513d11b 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -1,6 +1,6 @@ import colors from 'picocolors' import { - createOptimizeDepsRun, + runOptimizeDeps, getOptimizedDepPath, getHash, depsFromOptimizedDepInfo, @@ -12,13 +12,11 @@ import { depsLogString } from '.' import type { - DepOptimizationMetadata, DepOptimizationProcessing, OptimizedDepInfo, OptimizedDeps } from '.' import type { ViteDevServer } from '..' -import { resolveSSRExternal } from '../ssr/ssrExternal' /** * The amount to wait for requests to register newly found dependencies before triggering @@ -105,7 +103,7 @@ export function createOptimizedDeps( setTimeout(warmUp, 0) } - async function runOptimizer(ssr?: boolean) { + async function runOptimizer(isRerun = false) { // Ensure that rerun is called sequentially enqueuedRerun = undefined currentlyProcessing = true @@ -149,28 +147,23 @@ export function createOptimizedDeps( // dependencies will be asigned this promise from this point depOptimizationProcessing = newDepOptimizationProcessing() - let newData: DepOptimizationMetadata | null = null - try { - const processingResult = await createOptimizeDepsRun( - config, - newDeps, - metadata, - ssr - ) + const processingResult = await runOptimizeDeps(config, newDeps) - const commitProcessing = () => { - processingResult.commit() + const newData = processingResult.metadata - newData = processingResult.metadata + // After a re-optimization, if the internal bundled chunks change a full page reload + // is required. If the files are stable, we can avoid the reload that is expensive + // for large applications. Comparing their fileHash we can find out if it is safe to + // keep the current browser state. + const needsReload = Object.keys(metadata.optimized).some((dep) => { + return ( + metadata.optimized[dep]?.fileHash !== newData.optimized[dep]?.fileHash + ) + }) - // update ssr externals - if (ssr) { - server._ssrExternals = resolveSSRExternal( - server.config, - Object.keys(newData.optimized) - ) - } + const commitProcessing = () => { + processingResult.commit() // While optimizeDeps is running, new missing deps may be discovered, // in which case they will keep being added to metadata.discovered @@ -180,6 +173,19 @@ export function createOptimizedDeps( } } + // If we don't need to reload the page, we need to keep browserHash stable + if (isRerun && !needsReload) { + newData.browserHash = metadata.browserHash + for (const dep in newData.optimized) { + newData.optimized[dep].browserHash = ( + metadata.optimized[dep] || metadata.discovered[dep] + ).browserHash + } + for (const dep in newData.chunks) { + newData.chunks[dep].browserHash = metadata.browserHash + } + } + // Commit hash and needsInterop changes to the discovered deps info // object. Allow for code to await for the discovered processing promise // and use the information in the same object @@ -190,15 +196,15 @@ export function createOptimizedDeps( discovered.browserHash = optimized.browserHash discovered.fileHash = optimized.fileHash discovered.needsInterop = optimized.needsInterop + discovered.processing = undefined } } metadata = optimizedDeps.metadata = newData - resolveEnqueuedProcessingPromises() } - if (!processingResult.alteredFiles) { + if (!needsReload) { commitProcessing() logger.info(colors.green(`✨ dependencies pre-bundled...`), { @@ -261,7 +267,7 @@ export function createOptimizedDeps( }) } - async function rerun(ssr?: boolean) { + async function rerun() { // debounce time to wait for new missing deps finished, issue a new // optimization of deps (both old and newly found) once the previous // optimizeDeps processing is finished @@ -270,7 +276,7 @@ export function createOptimizedDeps( logger.info(colors.green(`new dependencies found: ${depsString}`), { timestamp: true }) - runOptimizer(ssr) + runOptimizer(true) } const discoveredTimestamp = Date.now() @@ -338,7 +344,7 @@ export function createOptimizedDeps( if (handle) clearTimeout(handle) handle = setTimeout(() => { handle = undefined - enqueuedRerun = () => rerun(ssr) + enqueuedRerun = rerun if (!currentlyProcessing) { enqueuedRerun() } diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts index 828deb00f9c837..8f6a48a28da463 100644 --- a/packages/vite/src/node/plugins/resolve.ts +++ b/packages/vite/src/node/plugins/resolve.ts @@ -646,8 +646,7 @@ export function tryNodeResolve( // get a resolved its optimized info const optimizedInfo = server._optimizedDeps!.registerMissingImport( id, - resolved, - ssr + resolved ) resolved = getOptimizedUrl(optimizedInfo) } diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index 03d4f399fcb9e1..024ed47191316c 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -262,6 +262,7 @@ export interface ViteDevServer { * @internal */ _ssrExternals: string[] | null + _ssrExternalsOutdated?: boolean /** * @internal */ @@ -361,21 +362,19 @@ export async function createServer( }, transformIndexHtml: null!, // to be immediately set async ssrLoadModule(url, opts?: { fixStacktrace?: boolean }) { - let configFileDependencies: string[] = [] - const optimizedDeps = server._optimizedDeps - if (optimizedDeps) { - await optimizedDeps.scanProcessing - const { metadata } = optimizedDeps - configFileDependencies = [ - ...Object.keys(metadata.optimized), - ...Object.keys(metadata.discovered) - ] + if (!server._ssrExternals || server._ssrExternalsOutdated) { + let knownImports: string[] = [] + const optimizedDeps = server._optimizedDeps + if (optimizedDeps) { + await optimizedDeps.scanProcessing + knownImports = [ + ...Object.keys(optimizedDeps.metadata.optimized), + ...Object.keys(optimizedDeps.metadata.discovered) + ] + } + server._ssrExternals = resolveSSRExternal(config, knownImports) + server._ssrExternalsOutdated = false } - - server._ssrExternals ||= resolveSSRExternal( - config, - configFileDependencies - ) return ssrLoadModule( url, server, @@ -430,6 +429,7 @@ export async function createServer( _optimizedDeps: null, _ssrExternals: null, + _ssrExternalsOutdated: false, _globImporters: Object.create(null), _restartPromise: null, _forceOptimizeOnRestart: false, From 93056e1cc9d0426ba1e984246dc1b170e3ef307c Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sun, 20 Mar 2022 17:50:01 +0100 Subject: [PATCH 07/18] chore: simplify parsing of metadata --- packages/vite/src/node/optimizer/index.ts | 116 ++++++++++------------ 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index bb93dd76e837bc..aefc47a8578446 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -662,31 +662,40 @@ function parseOptimizedDepsMetadata( jsonMetadata: string, depsCacheDir: string ) { - const metadata = JSON.parse(jsonMetadata, (key: string, value: string) => { - // Paths can be absolute or relative to the deps cache dir where - // the _metadata.json is located - if (key === 'file' || key === 'src') { - return normalizePath(path.resolve(depsCacheDir, value)) + const { hash, browserHash, optimized, chunks } = JSON.parse( + jsonMetadata, + (key: string, value: string) => { + // Paths can be absolute or relative to the deps cache dir where + // the _metadata.json is located + if (key === 'file' || key === 'src') { + return normalizePath(path.resolve(depsCacheDir, value)) + } + return value } - return value - }) - const { browserHash } = metadata - metadata.depInfoList = [] - for (const id of Object.keys(metadata.optimized)) { - const depInfo = metadata.optimized[id] - depInfo.id = id - depInfo.browserHash = browserHash - metadata.depInfoList.push(depInfo) + ) + const metadata = { + hash, + browserHash, + optimized: {}, + discovered: {}, + chunks: {}, + depInfoList: [] } - metadata.chunks ||= {} // Support missing chunks for back compat - for (const id of Object.keys(metadata.chunks)) { - const depInfo = metadata.chunks[id] - depInfo.id = id - depInfo.src = '' - depInfo.browserHash = browserHash - metadata.depInfoList.push(depInfo) + for (const id of Object.keys(optimized)) { + addOptimizedDepInfo(metadata, 'optimized', { + ...optimized[id], + id, + browserHash + }) + } + for (const id of Object.keys(chunks)) { + addOptimizedDepInfo(metadata, 'chunks', { + ...chunks[id], + id, + browserHash, + needsInterop: false + }) } - metadata.discovered = {} return metadata } @@ -700,49 +709,34 @@ function stringifyOptimizedDepsMetadata( metadata: DepOptimizationMetadata, depsCacheDir: string ) { + const { hash, browserHash, optimized, chunks } = metadata return JSON.stringify( - metadata, - (key: string, value: any) => { - if ( - key === 'discovered' || - key === 'processing' || - key === 'id' || - key === 'depInfoList' - ) { - return - } + { + hash, + browserHash, + optimized: Object.fromEntries( + Object.values(optimized).map( + ({ id, src, file, fileHash, needsInterop }) => [ + id, + { + src, + file, + fileHash, + needsInterop + } + ] + ) + ), + chunks: Object.fromEntries( + Object.values(chunks).map(({ id, file }) => [id, { file }]) + ) + }, + (key: string, value: string) => { + // Paths can be absolute or relative to the deps cache dir where + // the _metadata.json is located if (key === 'file' || key === 'src') { return normalizePath(path.relative(depsCacheDir, value)) } - if (key === 'optimized') { - // Only remove browserHash for individual dep info - const cleaned: Record = {} - for (const dep of Object.keys(value)) { - const { browserHash, ...c } = value[dep] - cleaned[dep] = c - } - return cleaned - } - if (key === 'optimized') { - return Object.keys(value).reduce( - (cleaned: Record, dep: string) => { - const { browserHash, ...c } = value[dep] - cleaned[dep] = c - return cleaned - }, - {} - ) - } - if (key === 'chunks') { - return Object.keys(value).reduce( - (cleaned: Record, dep: string) => { - const { browserHash, needsInterop, src, ...c } = value[dep] - cleaned[dep] = c - return cleaned - }, - {} - ) - } return value }, 2 From 6bf27f47bdda2569838ef3859eef43fff0bd69ed Mon Sep 17 00:00:00 2001 From: patak-dev Date: Sun, 20 Mar 2022 21:13:30 +0100 Subject: [PATCH 08/18] fix: windows --- packages/vite/src/node/optimizer/index.ts | 56 ++++++++--------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index aefc47a8578446..0ab45fb0a8d08f 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -477,19 +477,11 @@ export async function runOptimizeDeps( ) for (const id in depsInfo) { - const output = - meta.outputs[ - path.relative(process.cwd(), getProcessingDepPath(id, config)) - ] + const output = esbuildOutputFromId(meta.outputs, id, processingCacheDir) addOptimizedDepInfo(metadata, 'optimized', { ...depsInfo[id], - needsInterop: needsInterop( - id, - idToExports[id], - meta.outputs, - processingCacheDirOutputPath - ), + needsInterop: needsInterop(id, idToExports[id], output), // We only need to hash the output.imports in to check for stability, but adding the hash // and file path gives us a unique hash that may be useful for other things in the future fileHash: getHash( @@ -611,26 +603,16 @@ function getOptimizedBrowserHash(hash: string, deps: Record) { return getHash(hash + JSON.stringify(deps)) } -function getCachedDepFilePath(id: string, depsCacheDir: string) { - return normalizePath(path.resolve(depsCacheDir, flattenId(id) + '.js')) -} - export function getOptimizedDepPath(id: string, config: ResolvedConfig) { - return getCachedDepFilePath(id, getDepsCacheDir(config)) + return normalizePath( + path.resolve(getDepsCacheDir(config), flattenId(id) + '.js') + ) } export function getDepsCacheDir(config: ResolvedConfig) { return normalizePath(path.resolve(config.cacheDir, 'deps')) } -function getProcessingDepFilePath(id: string, processingCacheDir: string) { - return normalizePath(path.resolve(processingCacheDir, flattenId(id) + '.js')) -} - -function getProcessingDepPath(id: string, config: ResolvedConfig) { - return getProcessingDepFilePath(id, getProcessingDepsCacheDir(config)) -} - function getProcessingDepsCacheDir(config: ResolvedConfig) { return normalizePath(path.resolve(config.cacheDir, 'processing')) } @@ -743,6 +725,19 @@ function stringifyOptimizedDepsMetadata( ) } +function esbuildOutputFromId( + outputs: Record, + id: string, + cacheDirOutputPath: string +): any { + const flatId = flattenId(id) + '.js' + return outputs[ + normalizePath( + path.relative(process.cwd(), path.join(cacheDirOutputPath, flatId)) + ) + ] +} + // https://github.com/vitejs/vite/issues/1724#issuecomment-767619642 // a list of modules that pretends to be ESM but still uses `require`. // this causes esbuild to wrap them as CJS even when its entry appears to be ESM. @@ -751,8 +746,7 @@ const KNOWN_INTEROP_IDS = new Set(['moment']) function needsInterop( id: string, exportsData: ExportsData, - outputs: Record, - cacheDirOutputPath: string + output: any ): boolean { if (KNOWN_INTEROP_IDS.has(id)) { return true @@ -766,17 +760,7 @@ function needsInterop( // if a peer dependency used require() on a ESM dependency, esbuild turns the // ESM dependency's entry chunk into a single default export... detect // such cases by checking exports mismatch, and force interop. - const flatId = flattenId(id) + '.js' - let generatedExports: string[] | undefined - for (const output in outputs) { - if ( - normalizePath(output) === - normalizePath(path.join(cacheDirOutputPath, flatId)) - ) { - generatedExports = outputs[output].exports - break - } - } + const generatedExports: string[] = output.exports if ( !generatedExports || From 2b68162c197c5c99a4915b65289ad8349d643d18 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Mon, 21 Mar 2022 23:21:44 +0100 Subject: [PATCH 09/18] fix: laddle and vaadin start --- packages/vite/src/node/optimizer/index.ts | 57 +++++++++++-------- .../src/node/optimizer/registerMissing.ts | 55 +++++++++--------- packages/vite/src/node/optimizer/scan.ts | 10 +++- 3 files changed, 68 insertions(+), 54 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 0ab45fb0a8d08f..9bce3b81d4d459 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -191,12 +191,13 @@ export async function optimizeDeps( } export function createOptimizedDepsMetadata( - config: ResolvedConfig + config: ResolvedConfig, + timestamp?: string ): DepOptimizationMetadata { - const mainHash = getDepHash(config) + const hash = getDepHash(config) return { - hash: mainHash, - browserHash: mainHash, + hash, + browserHash: getOptimizedBrowserHash(hash, {}, timestamp), optimized: {}, chunks: {}, discovered: {}, @@ -233,8 +234,6 @@ export function loadCachedDepOptimizationMetadata( const depsCacheDir = getDepsCacheDir(config) - const mainHash = getDepHash(config) - if (!force) { let cachedMetadata: DepOptimizationMetadata | undefined try { @@ -245,7 +244,7 @@ export function loadCachedDepOptimizationMetadata( ) } catch (e) {} // hash is consistent, no need to re-bundle - if (cachedMetadata && cachedMetadata.hash === mainHash) { + if (cachedMetadata && cachedMetadata.hash === getDepHash(config)) { log('Hash is consistent. Skipping. Use --force to override.') // Nothing to commit or cancel as we are using the cache, we only // need to resolve the processing promise so requests can move on @@ -263,7 +262,8 @@ export function loadCachedDepOptimizationMetadata( */ export async function discoverProjectDependencies( - config: ResolvedConfig + config: ResolvedConfig, + timestamp?: string ): Promise> { const { deps, missing } = await scanImports(config) @@ -283,7 +283,11 @@ export async function discoverProjectDependencies( await addManuallyIncludedOptimizeDeps(deps, config) - const browserHash = getOptimizedBrowserHash(getDepHash(config), deps) + const browserHash = getOptimizedBrowserHash( + getDepHash(config), + deps, + timestamp + ) const discovered: Record = {} for (const id in deps) { const entry = deps[id] @@ -333,8 +337,6 @@ export async function runOptimizeDeps( const depsCacheDir = getDepsCacheDir(config) const processingCacheDir = getProcessingDepsCacheDir(config) - const mainHash = getDepHash(config) - // Create a temporal directory so we don't need to delete optimized deps // until they have been processed. This also avoids leaving the deps cache // directory in a corrupted state if there is an error @@ -354,7 +356,7 @@ export async function runOptimizeDeps( const metadata = createOptimizedDepsMetadata(config) metadata.browserHash = getOptimizedBrowserHash( - mainHash, + metadata.hash, depsFromOptimizedDepInfo(depsInfo) ) @@ -513,6 +515,9 @@ export async function runOptimizeDeps( } } + const dataPath = path.join(processingCacheDir, '_metadata.json') + writeFile(dataPath, stringifyOptimizedDepsMetadata(metadata, depsCacheDir)) + debug(`deps bundled in ${(performance.now() - start).toFixed(2)}ms`) return { @@ -525,10 +530,8 @@ export async function runOptimizeDeps( } function commitProcessingDepsCacheSync() { - // Rewire the file paths from the temporal processing dir to the final deps cache dir - const dataPath = path.join(processingCacheDir, '_metadata.json') - writeFile(dataPath, stringifyOptimizedDepsMetadata(metadata, depsCacheDir)) // Processing is done, we can now replace the depsCacheDir with processingCacheDir + // Rewire the file paths from the temporal processing dir to the final deps cache dir removeDirSync(depsCacheDir) fs.renameSync(processingCacheDir, depsCacheDir) } @@ -595,14 +598,6 @@ export function depsFromOptimizedDepInfo( ) } -export function getHash(text: string) { - return createHash('sha256').update(text).digest('hex').substring(0, 8) -} - -function getOptimizedBrowserHash(hash: string, deps: Record) { - return getHash(hash + JSON.stringify(deps)) -} - export function getOptimizedDepPath(id: string, config: ResolvedConfig) { return normalizePath( path.resolve(getDepsCacheDir(config), flattenId(id) + '.js') @@ -777,7 +772,7 @@ function isSingleDefaultExport(exports: readonly string[]) { const lockfileFormats = ['package-lock.json', 'yarn.lock', 'pnpm-lock.yaml'] -function getDepHash(config: ResolvedConfig): string { +export function getDepHash(config: ResolvedConfig): string { let content = lookupFile(config.root, lockfileFormats) || '' // also take config into account // only a subset of config options that can affect dep optimization @@ -808,7 +803,19 @@ function getDepHash(config: ResolvedConfig): string { return value } ) - return createHash('sha256').update(content).digest('hex').substring(0, 8) + return getHash(content) +} + +function getOptimizedBrowserHash( + hash: string, + deps: Record, + timestamp = '' +) { + return getHash(hash + JSON.stringify(deps) + timestamp) +} + +export function getHash(text: string) { + return createHash('sha256').update(text).digest('hex').substring(0, 8) } export function optimizedDepInfoFromId( diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 5795576513d11b..31fb3fcb6159ea 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -24,17 +24,17 @@ import type { ViteDevServer } from '..' */ const debounceMs = 100 -export function createOptimizedDeps( - server: ViteDevServer, - asCommand = false -): OptimizedDeps { +export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { const { config } = server const { logger } = config + const sessionTimestamp = Date.now().toString() + const cachedMetadata = loadCachedDepOptimizationMetadata(config) const optimizedDeps: OptimizedDeps = { - metadata: cachedMetadata || createOptimizedDepsMetadata(config), + metadata: + cachedMetadata || createOptimizedDepsMetadata(config, sessionTimestamp), registerMissingImport } @@ -70,7 +70,10 @@ export function createOptimizedDeps( const { metadata } = optimizedDeps - const discovered = await discoverProjectDependencies(config) + const discovered = await discoverProjectDependencies( + config, + sessionTimestamp + ) // Respect the scan phase discover order to improve reproducibility for (const depInfo of Object.values(discovered)) { @@ -80,8 +83,6 @@ export function createOptimizedDeps( }) } - // This is auto run on server start - let the user know that we are - // pre-optimizing deps const depsString = depsLogString(config, Object.keys(discovered)) logger.info(colors.green(`dependencies found: ${depsString}`), { timestamp: true @@ -103,7 +104,7 @@ export function createOptimizedDeps( setTimeout(warmUp, 0) } - async function runOptimizer(isRerun = false) { + async function runOptimizer() { // Ensure that rerun is called sequentially enqueuedRerun = undefined currentlyProcessing = true @@ -156,40 +157,43 @@ export function createOptimizedDeps( // is required. If the files are stable, we can avoid the reload that is expensive // for large applications. Comparing their fileHash we can find out if it is safe to // keep the current browser state. - const needsReload = Object.keys(metadata.optimized).some((dep) => { - return ( - metadata.optimized[dep]?.fileHash !== newData.optimized[dep]?.fileHash - ) - }) + const needsReload = + metadata.hash !== newData.hash || + Object.keys(metadata.optimized).some((dep) => { + return ( + metadata.optimized[dep]?.fileHash !== + newData.optimized[dep]?.fileHash + ) + }) const commitProcessing = () => { processingResult.commit() // While optimizeDeps is running, new missing deps may be discovered, // in which case they will keep being added to metadata.discovered - for (const id of Object.keys(metadata.discovered)) { + for (const id in metadata.discovered) { if (!newData.optimized[id]) { addOptimizedDepInfo(newData, 'discovered', metadata.discovered[id]) } } - // If we don't need to reload the page, we need to keep browserHash stable - if (isRerun && !needsReload) { + // If we don't reload the page, we need to keep browserHash stable + if (!needsReload) { newData.browserHash = metadata.browserHash + for (const dep in newData.chunks) { + newData.chunks[dep].browserHash = metadata.browserHash + } for (const dep in newData.optimized) { newData.optimized[dep].browserHash = ( metadata.optimized[dep] || metadata.discovered[dep] ).browserHash } - for (const dep in newData.chunks) { - newData.chunks[dep].browserHash = metadata.browserHash - } } // Commit hash and needsInterop changes to the discovered deps info // object. Allow for code to await for the discovered processing promise // and use the information in the same object - for (const o of Object.keys(newData.optimized)) { + for (const o in newData.optimized) { const discovered = metadata.discovered[o] if (discovered) { const optimized = newData.optimized[o] @@ -276,21 +280,16 @@ export function createOptimizedDeps( logger.info(colors.green(`new dependencies found: ${depsString}`), { timestamp: true }) - runOptimizer(true) + runOptimizer() } - const discoveredTimestamp = Date.now() - function getDiscoveredBrowserHash( hash: string, deps: Record, missing: Record ) { return getHash( - hash + - JSON.stringify(deps) + - JSON.stringify(missing) + - discoveredTimestamp + hash + JSON.stringify(deps) + JSON.stringify(missing) + sessionTimestamp ) } diff --git a/packages/vite/src/node/optimizer/scan.ts b/packages/vite/src/node/optimizer/scan.ts index d0da3664e98bdd..c6913e5465a88a 100644 --- a/packages/vite/src/node/optimizer/scan.ts +++ b/packages/vite/src/node/optimizer/scan.ts @@ -123,11 +123,19 @@ export async function scanImports(config: ResolvedConfig): Promise<{ debug(`Scan completed in ${(performance.now() - start).toFixed(2)}ms:`, deps) return { - deps, + // Ensure a fixed order so hashes are stable and improve logs + deps: orderedDependencies(deps), missing } } +function orderedDependencies(deps: Record) { + const depsList = Object.entries(deps) + // Ensure the same browserHash for the same set of dependencies + depsList.sort((a, b) => a[0].localeCompare(b[0])) + return Object.fromEntries(depsList) +} + function globEntries(pattern: string | string[], config: ResolvedConfig) { return glob(pattern, { cwd: config.root, From 6519a71d0ac53cbb47144ec4b3d7e29317ab1355 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 08:59:36 +0100 Subject: [PATCH 10/18] chore: apply suggestions --- packages/vite/src/node/optimizer/index.ts | 24 +++++++------------ .../src/node/optimizer/registerMissing.ts | 6 ++--- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 9bce3b81d4d459..deb2e3b780b448 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -33,7 +33,7 @@ export type ExportsData = ReturnType & { hasReExports?: true } -export type OptimizedDeps = { +export interface OptimizedDeps { metadata: DepOptimizationMetadata scanProcessing?: Promise registerMissingImport: (id: string, resolved: string) => OptimizedDepInfo @@ -180,7 +180,7 @@ export async function optimizeDeps( } const depsInfo = await discoverProjectDependencies(config) - const depsString = depsLogString(config, Object.keys(depsInfo)) + const depsString = depsLogString(Object.keys(depsInfo)) config.logger.info(colors.green(`Optimizing dependencies:\n ${depsString}`)) const result = await runOptimizeDeps(config, depsInfo) @@ -237,9 +237,9 @@ export function loadCachedDepOptimizationMetadata( if (!force) { let cachedMetadata: DepOptimizationMetadata | undefined try { - const cachedMetadataPatah = path.join(depsCacheDir, '_metadata.json') + const cachedMetadataPath = path.join(depsCacheDir, '_metadata.json') cachedMetadata = parseOptimizedDepsMetadata( - fs.readFileSync(cachedMetadataPatah, 'utf-8'), + fs.readFileSync(cachedMetadataPath, 'utf-8'), depsCacheDir ) } catch (e) {} @@ -260,7 +260,6 @@ export function loadCachedDepOptimizationMetadata( * Initial optimizeDeps at server start. Perform a fast scan using esbuild to * find deps to pre-bundle and include user hard-coded dependencies */ - export async function discoverProjectDependencies( config: ResolvedConfig, timestamp?: string @@ -301,24 +300,19 @@ export async function discoverProjectDependencies( return discovered } -export function depsLogString( - config: ResolvedConfig, - qualifiedIds: string[] -): string { - let depsString: string +export function depsLogString(qualifiedIds: string[]): string { if (isDebugEnabled) { - depsString = colors.yellow(qualifiedIds.join(`\n `)) + return colors.yellow(qualifiedIds.join(`\n `)) } else { const total = qualifiedIds.length const maxListed = 5 const listed = Math.min(total, maxListed) const extra = Math.max(0, total - maxListed) - depsString = colors.yellow( + return colors.yellow( qualifiedIds.slice(0, listed).join(`, `) + (extra > 0 ? `, ...and ${extra} more` : ``) ) } - return depsString } /** @@ -741,7 +735,7 @@ const KNOWN_INTEROP_IDS = new Set(['moment']) function needsInterop( id: string, exportsData: ExportsData, - output: any + output: { exports: string[] } ): boolean { if (KNOWN_INTEROP_IDS.has(id)) { return true @@ -814,7 +808,7 @@ function getOptimizedBrowserHash( return getHash(hash + JSON.stringify(deps) + timestamp) } -export function getHash(text: string) { +export function getHash(text: string): string { return createHash('sha256').update(text).digest('hex').substring(0, 8) } diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 31fb3fcb6159ea..08fdbed02777a2 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -62,7 +62,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { const scanPhaseProcessing = newDepOptimizationProcessing() optimizedDeps.scanProcessing = scanPhaseProcessing.promise - const warmUp = async function () { + const warmUp = async () => { try { logger.info(colors.green(`scanning for dependencies...`), { timestamp: true @@ -83,7 +83,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { }) } - const depsString = depsLogString(config, Object.keys(discovered)) + const depsString = depsLogString(Object.keys(discovered)) logger.info(colors.green(`dependencies found: ${depsString}`), { timestamp: true }) @@ -276,7 +276,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { // optimization of deps (both old and newly found) once the previous // optimizeDeps processing is finished const deps = Object.keys(optimizedDeps.metadata.discovered) - const depsString = depsLogString(config, deps) + const depsString = depsLogString(deps) logger.info(colors.green(`new dependencies found: ${depsString}`), { timestamp: true }) From ecbda41ec366fee0239aaffe4f05a3f348e68f70 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 09:06:20 +0100 Subject: [PATCH 11/18] chore: asCommand log --- packages/vite/src/node/optimizer/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index deb2e3b780b448..b995578029ea16 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -170,6 +170,8 @@ export async function optimizeDeps( force = config.server.force, asCommand = false ): Promise { + const log = asCommand ? config.logger.info : debug + const cachedMetadata = loadCachedDepOptimizationMetadata( config, force, @@ -181,7 +183,7 @@ export async function optimizeDeps( const depsInfo = await discoverProjectDependencies(config) const depsString = depsLogString(Object.keys(depsInfo)) - config.logger.info(colors.green(`Optimizing dependencies:\n ${depsString}`)) + log(colors.green(`Optimizing dependencies:\n ${depsString}`)) const result = await runOptimizeDeps(config, depsInfo) From 2d630a3526798c0dd595a991f07a113e0ce445f8 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 09:53:22 +0100 Subject: [PATCH 12/18] chore: remove unneeded ssr code --- packages/vite/src/node/optimizer/esbuildDepPlugin.ts | 5 ++--- packages/vite/src/node/server/index.ts | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts index 9ebd59b76b12fd..4303be0ec876e7 100644 --- a/packages/vite/src/node/optimizer/esbuildDepPlugin.ts +++ b/packages/vite/src/node/optimizer/esbuildDepPlugin.ts @@ -37,8 +37,7 @@ const externalTypes = [ export function esbuildDepPlugin( qualified: Record, exportsData: Record, - config: ResolvedConfig, - ssr?: boolean + config: ResolvedConfig ): Plugin { // remove optimizable extensions from `externalTypes` list const allExternalTypes = config.optimizeDeps.extensions @@ -73,7 +72,7 @@ export function esbuildDepPlugin( _importer = importer in qualified ? qualified[importer] : importer } const resolver = kind.startsWith('require') ? _resolveRequire : _resolve - return resolver(id, _importer, undefined, ssr) + return resolver(id, _importer, undefined) } return { diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index 024ed47191316c..e8d4c3f1e5626d 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -262,7 +262,6 @@ export interface ViteDevServer { * @internal */ _ssrExternals: string[] | null - _ssrExternalsOutdated?: boolean /** * @internal */ @@ -362,7 +361,7 @@ export async function createServer( }, transformIndexHtml: null!, // to be immediately set async ssrLoadModule(url, opts?: { fixStacktrace?: boolean }) { - if (!server._ssrExternals || server._ssrExternalsOutdated) { + if (!server._ssrExternals) { let knownImports: string[] = [] const optimizedDeps = server._optimizedDeps if (optimizedDeps) { @@ -373,7 +372,6 @@ export async function createServer( ] } server._ssrExternals = resolveSSRExternal(config, knownImports) - server._ssrExternalsOutdated = false } return ssrLoadModule( url, @@ -429,7 +427,6 @@ export async function createServer( _optimizedDeps: null, _ssrExternals: null, - _ssrExternalsOutdated: false, _globImporters: Object.create(null), _restartPromise: null, _forceOptimizeOnRestart: false, From 8a54e91c14b70c2db86e18fbbc86af7e8b683413 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 10:04:05 +0100 Subject: [PATCH 13/18] chore: explicit fileHash comparison --- packages/vite/src/node/optimizer/index.ts | 9 ++++++++- packages/vite/src/node/optimizer/registerMissing.ts | 3 +-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index b995578029ea16..652a51e4544fb0 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -634,7 +634,7 @@ export function createIsOptimizedDepUrl(config: ResolvedConfig) { function parseOptimizedDepsMetadata( jsonMetadata: string, depsCacheDir: string -) { +): DepOptimizationMetadata | undefined { const { hash, browserHash, optimized, chunks } = JSON.parse( jsonMetadata, (key: string, value: string) => { @@ -646,6 +646,13 @@ function parseOptimizedDepsMetadata( return value } ) + if ( + !chunks || + Object.values(optimized).some((depInfo: any) => !depInfo.fileHash) + ) { + // outdated _metadata.json version, ignore + return + } const metadata = { hash, browserHash, diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 08fdbed02777a2..243213054a7c45 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -161,8 +161,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { metadata.hash !== newData.hash || Object.keys(metadata.optimized).some((dep) => { return ( - metadata.optimized[dep]?.fileHash !== - newData.optimized[dep]?.fileHash + metadata.optimized[dep].fileHash !== newData.optimized[dep].fileHash ) }) From e0c54f85fc6eb84839d58b82ca12976dbf0f7814 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 10:07:51 +0100 Subject: [PATCH 14/18] test: increase cli-module timeout --- packages/playground/cli-module/__tests__/serve.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playground/cli-module/__tests__/serve.js b/packages/playground/cli-module/__tests__/serve.js index 1cda05f0adc21a..2b354f566524bf 100644 --- a/packages/playground/cli-module/__tests__/serve.js +++ b/packages/playground/cli-module/__tests__/serve.js @@ -78,7 +78,7 @@ exports.serve = async function serve(root, isProd) { const timeoutError = `server process still alive after 3s` try { killProcess(serverProcess) - await resolvedOrTimeout(serverProcess, 3000, timeoutError) + await resolvedOrTimeout(serverProcess, 10000, timeoutError) } catch (e) { if (e === timeoutError || (!serverProcess.killed && !isWindows)) { collectErrorStreams('server', e) From 020b25b092a590c194e39e63c0d0033dd0b640da Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 10:25:43 +0100 Subject: [PATCH 15/18] chore: mark scan as internal --- packages/vite/src/node/plugin.ts | 7 ++++++- packages/vite/src/node/server/pluginContainer.ts | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/vite/src/node/plugin.ts b/packages/vite/src/node/plugin.ts index 1d3cfb2d3bfcdc..c78be6511a6be5 100644 --- a/packages/vite/src/node/plugin.ts +++ b/packages/vite/src/node/plugin.ts @@ -121,7 +121,12 @@ export interface Plugin extends RollupPlugin { this: PluginContext, source: string, importer: string | undefined, - options: { custom?: CustomPluginOptions; ssr?: boolean; scan?: boolean } + options: { + custom?: CustomPluginOptions + ssr?: boolean + // @internal + scan?: boolean + } ): Promise | ResolveIdResult load?( this: PluginContext, diff --git a/packages/vite/src/node/server/pluginContainer.ts b/packages/vite/src/node/server/pluginContainer.ts index c32dd7db89127a..6a5f78a14054c9 100644 --- a/packages/vite/src/node/server/pluginContainer.ts +++ b/packages/vite/src/node/server/pluginContainer.ts @@ -90,6 +90,7 @@ export interface PluginContainer { options?: { skip?: Set ssr?: boolean + // @internal scan?: boolean } ): Promise From 26e726e38272f96bac999bf9f8a143bd46031565 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 10:49:40 +0100 Subject: [PATCH 16/18] chore: improve logs --- packages/vite/src/node/optimizer/index.ts | 3 +- .../src/node/optimizer/registerMissing.ts | 38 +++++++++++++------ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 652a51e4544fb0..cefa24133db529 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -21,7 +21,8 @@ import { scanImports } from './scan' import { transformWithEsbuild } from '../plugins/esbuild' import { performance } from 'perf_hooks' -const debug = createDebugger('vite:deps') +export const debuggerViteDeps = createDebugger('vite:deps') +const debug = debuggerViteDeps const isDebugEnabled = _debug('vite:deps').enabled const jsExtensionRE = /\.js$/i diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index 243213054a7c45..beeaf083295455 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -9,7 +9,8 @@ import { createOptimizedDepsMetadata, addOptimizedDepInfo, discoverProjectDependencies, - depsLogString + depsLogString, + debuggerViteDeps as debug } from '.' import type { DepOptimizationProcessing, @@ -64,7 +65,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { const warmUp = async () => { try { - logger.info(colors.green(`scanning for dependencies...`), { + debug(colors.green(`scanning for dependencies...`), { timestamp: true }) @@ -83,10 +84,14 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { }) } - const depsString = depsLogString(Object.keys(discovered)) - logger.info(colors.green(`dependencies found: ${depsString}`), { - timestamp: true - }) + debug( + colors.green( + `dependencies found: ${depsLogString(Object.keys(discovered))}` + ), + { + timestamp: true + } + ) scanPhaseProcessing.resolve() optimizedDeps.scanProcessing = undefined @@ -203,14 +208,23 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { } } + const newDeps = Object.keys(newData.optimized).filter( + (dep) => !metadata.optimized[dep] + ) + config.logger.info( + colors.green(`✨ dependencies optimized: ${depsLogString(newDeps)}`), + { + timestamp: true + } + ) + metadata = optimizedDeps.metadata = newData resolveEnqueuedProcessingPromises() } if (!needsReload) { commitProcessing() - - logger.info(colors.green(`✨ dependencies pre-bundled...`), { + debug(colors.green(`✨ previous optimized dependencies unchanged`), { timestamp: true }) } else { @@ -221,7 +235,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { // once a rerun is committed processingResult.cancel() - logger.info( + debug( colors.green( `✨ delaying reload as new dependencies have been found...` ), @@ -233,7 +247,9 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { commitProcessing() logger.info( - colors.green(`✨ dependencies updated, reloading page...`), + colors.green( + `✨ previous optimized dependencies have changed, reloading page` + ), { timestamp: true } @@ -276,7 +292,7 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { // optimizeDeps processing is finished const deps = Object.keys(optimizedDeps.metadata.discovered) const depsString = depsLogString(deps) - logger.info(colors.green(`new dependencies found: ${depsString}`), { + debug(colors.green(`new dependencies found: ${depsString}`), { timestamp: true }) runOptimizer() From e55d8a9f5f3e10042dafd6e1b093c548a23ea686 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 13:52:31 +0100 Subject: [PATCH 17/18] feat: batch dependencies optimized logs --- .../src/node/optimizer/registerMissing.ts | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/vite/src/node/optimizer/registerMissing.ts b/packages/vite/src/node/optimizer/registerMissing.ts index beeaf083295455..65508ac5ae5155 100644 --- a/packages/vite/src/node/optimizer/registerMissing.ts +++ b/packages/vite/src/node/optimizer/registerMissing.ts @@ -1,4 +1,5 @@ import colors from 'picocolors' +import _debug from 'debug' import { runOptimizeDeps, getOptimizedDepPath, @@ -19,6 +20,8 @@ import type { } from '.' import type { ViteDevServer } from '..' +const isDebugEnabled = _debug('vite:deps').enabled + /** * The amount to wait for requests to register newly found dependencies before triggering * a re-bundle + page reload @@ -42,6 +45,18 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { let handle: NodeJS.Timeout | undefined let newDepsDiscovered = false + let newDepsToLog: string[] = [] + let newDepsToLogHandle: NodeJS.Timeout | undefined + const logNewDeps = () => { + config.logger.info( + colors.green(`✨ dependencies optimized: ${depsLogString(newDepsToLog)}`), + { + timestamp: true + } + ) + newDepsToLog = [] + } + let depOptimizationProcessing = newDepOptimizationProcessing() let depOptimizationProcessingQueue: DepOptimizationProcessing[] = [] const resolveEnqueuedProcessingPromises = () => { @@ -208,14 +223,10 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { } } - const newDeps = Object.keys(newData.optimized).filter( - (dep) => !metadata.optimized[dep] - ) - config.logger.info( - colors.green(`✨ dependencies optimized: ${depsLogString(newDeps)}`), - { - timestamp: true - } + newDepsToLog.push( + ...Object.keys(newData.optimized).filter( + (dep) => !metadata.optimized[dep] + ) ) metadata = optimizedDeps.metadata = newData @@ -224,9 +235,19 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { if (!needsReload) { commitProcessing() - debug(colors.green(`✨ previous optimized dependencies unchanged`), { - timestamp: true - }) + + if (isDebugEnabled) { + logNewDeps() + debug(colors.green(`✨ previous optimized dependencies unchanged`), { + timestamp: true + }) + } else { + if (newDepsToLogHandle) clearTimeout(newDepsToLogHandle) + newDepsToLogHandle = setTimeout(() => { + newDepsToLogHandle = undefined + logNewDeps() + }, 2 * debounceMs) + } } else { if (newDepsDiscovered) { // There are newly discovered deps, and another rerun is about to be @@ -246,6 +267,10 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { } else { commitProcessing() + if (newDepsToLogHandle) clearTimeout(newDepsToLogHandle) + newDepsToLogHandle = undefined + logNewDeps() + logger.info( colors.green( `✨ previous optimized dependencies have changed, reloading page` @@ -356,6 +381,8 @@ export function createOptimizedDeps(server: ViteDevServer): OptimizedDeps { // the running next optimizeDeps enqueuedRerun = undefined if (handle) clearTimeout(handle) + if (newDepsToLogHandle) clearTimeout(newDepsToLogHandle) + newDepsToLogHandle = undefined handle = setTimeout(() => { handle = undefined enqueuedRerun = rerun From df532033d0d740a4b902c429fc3870fb97fd2733 Mon Sep 17 00:00:00 2001 From: patak-dev Date: Tue, 22 Mar 2022 13:55:05 +0100 Subject: [PATCH 18/18] chore: apply blu suggestion about @internal --- packages/vite/src/node/plugin.ts | 4 +++- packages/vite/src/node/server/pluginContainer.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/vite/src/node/plugin.ts b/packages/vite/src/node/plugin.ts index c78be6511a6be5..354b246dd9f182 100644 --- a/packages/vite/src/node/plugin.ts +++ b/packages/vite/src/node/plugin.ts @@ -124,7 +124,9 @@ export interface Plugin extends RollupPlugin { options: { custom?: CustomPluginOptions ssr?: boolean - // @internal + /** + * @internal + */ scan?: boolean } ): Promise | ResolveIdResult diff --git a/packages/vite/src/node/server/pluginContainer.ts b/packages/vite/src/node/server/pluginContainer.ts index 6a5f78a14054c9..848bb0657ddfed 100644 --- a/packages/vite/src/node/server/pluginContainer.ts +++ b/packages/vite/src/node/server/pluginContainer.ts @@ -90,7 +90,9 @@ export interface PluginContainer { options?: { skip?: Set ssr?: boolean - // @internal + /** + * @internal + */ scan?: boolean } ): Promise