From 3f3024c154a8f9a9aff17311a6fbdc9973b94ed8 Mon Sep 17 00:00:00 2001 From: patak Date: Sat, 25 Mar 2023 17:48:22 +0100 Subject: [PATCH 1/5] fix: avoid temporal optimize deps dirs --- packages/vite/src/node/optimizer/index.ts | 170 +++++++++------------- packages/vite/src/node/server/index.ts | 5 - packages/vite/src/node/utils.ts | 93 ------------ 3 files changed, 70 insertions(+), 198 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 496664baf8897a..44130387885f73 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -20,11 +20,8 @@ import { lookupFile, normalizeId, normalizePath, - removeDir, removeLeadingSlash, - renameDir, tryStatSync, - writeFile, } from '../utils' import { transformWithEsbuild } from '../plugins/esbuild' import { ESBUILD_MODULES_TARGET } from '../constants' @@ -164,6 +161,9 @@ export interface DepOptimizationResult { * to be able to discard the result */ commit: () => Promise + /** + * @deprecated noop + */ cancel: () => void } @@ -474,23 +474,6 @@ export function runOptimizeDeps( } const depsCacheDir = getDepsCacheDir(resolvedConfig, ssr) - const processingCacheDir = getProcessingDepsCacheDir(resolvedConfig, ssr) - - // Create a temporal directory so we don't need to delete optimized deps - // until they have been processed. This also avoids leaving the deps cache - // directory in a corrupted state if there is an error - if (fs.existsSync(processingCacheDir)) { - emptyDir(processingCacheDir) - } else { - fs.mkdirSync(processingCacheDir, { recursive: true }) - } - - // a hint for Node.js - // all files in the cache directory should be recognized as ES modules - writeFile( - path.resolve(processingCacheDir, 'package.json'), - JSON.stringify({ type: 'module' }), - ) const metadata = initDepsOptimizerMetadata(config, ssr) @@ -505,38 +488,16 @@ export function runOptimizeDeps( const qualifiedIds = Object.keys(depsInfo) - let cleaned = false - const cleanUp = () => { - if (!cleaned) { - cleaned = true - // No need to wait, we can clean up in the background because temp folders - // are unique per run - fsp.rm(processingCacheDir, { recursive: true, force: true }).catch(() => { - // Ignore errors - }) - } - } - const createProcessingResult = () => ({ + const createEmptyProcessingResult = () => ({ metadata, - async commit() { - if (cleaned) { - throw new Error( - `Vite Internal Error: Can't commit optimizeDeps processing result, it has already been cancelled.`, - ) - } - // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` - // Processing is done, we can now replace the depsCacheDir with processingCacheDir - // Rewire the file paths from the temporal processing dir to the final deps cache dir - await removeDir(depsCacheDir) - await renameDir(processingCacheDir, depsCacheDir) - }, - cancel: cleanUp, + commit: async () => {}, + cancel: async () => {}, }) if (!qualifiedIds.length) { return { - cancel: async () => cleanUp(), - result: Promise.resolve(createProcessingResult()), + result: Promise.resolve(createEmptyProcessingResult()), + cancel: async () => {}, } } @@ -546,11 +507,11 @@ export function runOptimizeDeps( resolvedConfig, depsInfo, ssr, - processingCacheDir, + depsCacheDir, optimizerContext, ) - const result = preparedRun.then(({ context, idToExports }) => { + const runResult = preparedRun.then(({ context, idToExports }) => { function disposeContext() { return context?.dispose().catch((e) => { config.logger.error('Failed to dispose esbuild context', { error: e }) @@ -558,7 +519,7 @@ export function runOptimizeDeps( } if (!context || optimizerContext.cancelled) { disposeContext() - return createProcessingResult() + return createEmptyProcessingResult() } return context @@ -569,15 +530,11 @@ export function runOptimizeDeps( // the paths in `meta.outputs` are relative to `process.cwd()` const processingCacheDirOutputPath = path.relative( process.cwd(), - processingCacheDir, + depsCacheDir, ) for (const id in depsInfo) { - const output = esbuildOutputFromId( - meta.outputs, - id, - processingCacheDir, - ) + const output = esbuildOutputFromId(meta.outputs, id, depsCacheDir) const { exportsData, ...info } = depsInfo[id] addOptimizedDepInfo(metadata, 'optimized', { @@ -624,23 +581,58 @@ export function runOptimizeDeps( } } - const dataPath = path.join(processingCacheDir, '_metadata.json') - writeFile( - dataPath, - stringifyDepsOptimizerMetadata(metadata, depsCacheDir), - ) - debug( `Dependencies bundled in ${(performance.now() - start).toFixed(2)}ms`, ) - return createProcessingResult() + return { + metadata, + async commit() { + // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` + // Processing is done, we can now replace the depsCacheDir with processingCacheDir + // Rewire the file paths from the temporal processing dir to the final deps cache dir + + if (!fs.existsSync(depsCacheDir)) { + fs.mkdirSync(depsCacheDir, { recursive: true }) + } + + const files = [] + + // a hint for Node.js + // all files in the cache directory should be recognized as ES modules + files.push( + fsp.writeFile( + path.resolve(depsCacheDir, 'package.json'), + JSON.stringify({ type: 'module' }), + ), + ) + + const dataPath = path.join(depsCacheDir, '_metadata.json') + files.push( + fsp.writeFile( + dataPath, + stringifyDepsOptimizerMetadata(metadata, depsCacheDir), + ), + ) + + for (const outputFile of result.outputFiles!) { + files.push(fsp.writeFile(outputFile.path, outputFile.text)) + } + + await Promise.all(files) + + // Clean up old files in the background + cleanupDepsCacheStaleFiles(depsCacheDir, metadata) + }, + cancel: () => {}, + } }) + .catch((e) => { if (e.errors && e.message.includes('The build was canceled')) { // esbuild logs an error when cancelling, but this is expected so // return an empty result instead - return createProcessingResult() + return createEmptyProcessingResult() } throw e }) @@ -649,18 +641,13 @@ export function runOptimizeDeps( }) }) - result.catch(() => { - cleanUp() - }) - return { async cancel() { optimizerContext.cancelled = true const { context } = await preparedRun await context?.cancel() - cleanUp() }, - result, + result: runResult, } } @@ -760,6 +747,9 @@ async function prepareEsbuildOptimizerRun( absWorkingDir: process.cwd(), entryPoints: Object.keys(flatIdDeps), bundle: true, + // Don't write to disk, we'll only write the files if the build isn't invalidated + // by newly discovered dependencies + write: false, // We can't use platform 'neutral', as esbuild has custom handling // when the platform is 'node' or 'browser' that can't be emulated // by using mainFields and conditions @@ -934,15 +924,6 @@ export function getDepsCacheDir(config: ResolvedConfig, ssr: boolean): string { return getDepsCacheDirPrefix(config) + getDepsCacheSuffix(config, ssr) } -function getProcessingDepsCacheDir(config: ResolvedConfig, ssr: boolean) { - return ( - getDepsCacheDirPrefix(config) + - getDepsCacheSuffix(config, ssr) + - '_temp_' + - getHash(Date.now().toString()) - ) -} - export function getDepsCacheDirPrefix(config: ResolvedConfig): string { return normalizePath(path.resolve(config.cacheDir, 'deps')) } @@ -1306,28 +1287,17 @@ export async function optimizedDepNeedsInterop( return depInfo?.needsInterop } -const MAX_TEMP_DIR_AGE_MS = 24 * 60 * 60 * 1000 -export async function cleanupDepsCacheStaleDirs( - config: ResolvedConfig, -): Promise { - try { - const cacheDir = path.resolve(config.cacheDir) - if (fs.existsSync(cacheDir)) { - const dirents = await fsp.readdir(cacheDir, { withFileTypes: true }) - for (const dirent of dirents) { - if (dirent.isDirectory() && dirent.name.includes('_temp_')) { - const tempDirPath = path.resolve(config.cacheDir, dirent.name) - const stats = await fsp.stat(tempDirPath).catch((_) => null) - if ( - stats?.mtime && - Date.now() - stats.mtime.getTime() > MAX_TEMP_DIR_AGE_MS - ) { - await removeDir(tempDirPath) - } - } +async function cleanupDepsCacheStaleFiles( + depsCacheDir: string, + metadata: DepOptimizationMetadata, +) { + const files = await fsp.readdir(depsCacheDir) + for (const file of files) { + if (file !== 'package.json' && file !== '_metadata.json') { + const filePath = path.join(depsCacheDir, file) + if (!optimizedDepInfoFromFile(metadata, filePath)) { + fsp.unlink(filePath) } } - } catch (err) { - config.logger.error(err) } } diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts index 4644c06f2d379d..45427f4eccf506 100644 --- a/packages/vite/src/node/server/index.ts +++ b/packages/vite/src/node/server/index.ts @@ -35,7 +35,6 @@ import { cjsSsrResolveExternals } from '../ssr/ssrExternal' import { ssrFixStacktrace, ssrRewriteStacktrace } from '../ssr/ssrStacktrace' import { ssrTransform } from '../ssr/ssrTransform' import { - cleanupDepsCacheStaleDirs, getDepsOptimizer, initDepsOptimizer, initDevSsrDepsOptimizer, @@ -693,10 +692,6 @@ export async function createServer( await initServer() } - // Fire a clean up of stale cache dirs, in case old processes didn't - // terminate correctly. Don't await this promise - cleanupDepsCacheStaleDirs(config) - return server } diff --git a/packages/vite/src/node/utils.ts b/packages/vite/src/node/utils.ts index d1b3eb14cae7c8..ac7b9589532f4a 100644 --- a/packages/vite/src/node/utils.ts +++ b/packages/vite/src/node/utils.ts @@ -2,7 +2,6 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { createHash } from 'node:crypto' -import { promisify } from 'node:util' import { URL, URLSearchParams } from 'node:url' import { builtinModules, createRequire } from 'node:module' import { promises as dns } from 'node:dns' @@ -503,17 +502,6 @@ export function generateCodeFrame( return res.join('\n') } -export function writeFile( - filename: string, - content: string | Uint8Array, -): void { - const dir = path.dirname(filename) - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }) - } - fs.writeFileSync(filename, content) -} - export function isFileReadable(filename: string): boolean { try { fs.accessSync(filename, fs.constants.R_OK) @@ -582,18 +570,6 @@ export function copyDir(srcDir: string, destDir: string): void { } } -export const removeDir = isWindows - ? promisify(gracefulRemoveDir) - : function removeDirSync(dir: string) { - // when removing `.vite/deps`, if it doesn't exist, nodejs may also remove - // other directories within `.vite/`, including `.vite/deps_temp` (bug). - // workaround by checking for directory existence before removing for now. - if (fs.existsSync(dir)) { - fs.rmSync(dir, { recursive: true, force: true }) - } - } -export const renameDir = isWindows ? promisify(gracefulRename) : fs.renameSync - // `fs.realpathSync.native` resolves differently in Windows network drive, // causing file read errors. skip for now. // https://github.com/nodejs/node/issues/37737 @@ -963,75 +939,6 @@ export const requireResolveFromRootWithFallback = ( return _require.resolve(id, { paths }) } -// Based on node-graceful-fs - -// The ISC License -// Copyright (c) 2011-2022 Isaac Z. Schlueter, Ben Noordhuis, and Contributors -// https://github.com/isaacs/node-graceful-fs/blob/main/LICENSE - -// On Windows, A/V software can lock the directory, causing this -// to fail with an EACCES or EPERM if the directory contains newly -// created files. The original tried for up to 60 seconds, we only -// wait for 5 seconds, as a longer time would be seen as an error - -const GRACEFUL_RENAME_TIMEOUT = 5000 -function gracefulRename( - from: string, - to: string, - cb: (error: NodeJS.ErrnoException | null) => void, -) { - const start = Date.now() - let backoff = 0 - fs.rename(from, to, function CB(er) { - if ( - er && - (er.code === 'EACCES' || er.code === 'EPERM') && - Date.now() - start < GRACEFUL_RENAME_TIMEOUT - ) { - setTimeout(function () { - fs.stat(to, function (stater, st) { - if (stater && stater.code === 'ENOENT') fs.rename(from, to, CB) - else CB(er) - }) - }, backoff) - if (backoff < 100) backoff += 10 - return - } - if (cb) cb(er) - }) -} - -const GRACEFUL_REMOVE_DIR_TIMEOUT = 5000 -function gracefulRemoveDir( - dir: string, - cb: (error: NodeJS.ErrnoException | null) => void, -) { - const start = Date.now() - let backoff = 0 - fs.rm(dir, { recursive: true }, function CB(er) { - if (er) { - if ( - (er.code === 'ENOTEMPTY' || - er.code === 'EACCES' || - er.code === 'EPERM') && - Date.now() - start < GRACEFUL_REMOVE_DIR_TIMEOUT - ) { - setTimeout(function () { - fs.rm(dir, { recursive: true }, CB) - }, backoff) - if (backoff < 100) backoff += 10 - return - } - - if (er.code === 'ENOENT') { - er = null - } - } - - if (cb) cb(er) - }) -} - export function emptyCssComments(raw: string): string { return raw.replace(multilineCommentsRE, (s) => ' '.repeat(s.length)) } From 3c69081395936323350901358416692f8ec1735c Mon Sep 17 00:00:00 2001 From: patak Date: Sat, 25 Mar 2023 18:02:46 +0100 Subject: [PATCH 2/5] chore: windows --- packages/vite/src/node/optimizer/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 44130387885f73..119365fdb182ed 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -1295,7 +1295,7 @@ async function cleanupDepsCacheStaleFiles( for (const file of files) { if (file !== 'package.json' && file !== '_metadata.json') { const filePath = path.join(depsCacheDir, file) - if (!optimizedDepInfoFromFile(metadata, filePath)) { + if (!optimizedDepInfoFromFile(metadata, normalizePath(filePath))) { fsp.unlink(filePath) } } From 7a3e2b1f6e499a0a3b3d1454c56282ecec86a955 Mon Sep 17 00:00:00 2001 From: patak Date: Sun, 26 Mar 2023 11:10:04 +0200 Subject: [PATCH 3/5] chore: update --- packages/vite/src/node/optimizer/index.ts | 38 ++++++++++++++--------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 119365fdb182ed..29bba44ca431ed 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -588,24 +588,32 @@ export function runOptimizeDeps( return { metadata, async commit() { - // Write metadata file, delete `deps` folder and rename the `processing` folder to `deps` - // Processing is done, we can now replace the depsCacheDir with processingCacheDir - // Rewire the file paths from the temporal processing dir to the final deps cache dir + // Write this run of pre-bundled dependencies to the deps cache + let oldFilesNames: string[] if (!fs.existsSync(depsCacheDir)) { fs.mkdirSync(depsCacheDir, { recursive: true }) + oldFilesNames = [] + } else { + oldFilesNames = await fsp.readdir(depsCacheDir) } const files = [] // a hint for Node.js // all files in the cache directory should be recognized as ES modules - files.push( - fsp.writeFile( - path.resolve(depsCacheDir, 'package.json'), - JSON.stringify({ type: 'module' }), - ), + const packageJsonFilePath = path.resolve( + depsCacheDir, + 'package.json', ) + if (!fs.existsSync(packageJsonFilePath)) { + files.push( + fsp.writeFile( + packageJsonFilePath, + '{\n "type": "module"\n}\n', + ), + ) + } const dataPath = path.join(depsCacheDir, '_metadata.json') files.push( @@ -619,10 +627,10 @@ export function runOptimizeDeps( files.push(fsp.writeFile(outputFile.path, outputFile.text)) } - await Promise.all(files) - // Clean up old files in the background - cleanupDepsCacheStaleFiles(depsCacheDir, metadata) + cleanupDepsCacheStaleFiles(oldFilesNames, depsCacheDir, metadata) + + await Promise.all(files) }, cancel: () => {}, } @@ -1288,13 +1296,13 @@ export async function optimizedDepNeedsInterop( } async function cleanupDepsCacheStaleFiles( + oldFilesNames: string[], depsCacheDir: string, metadata: DepOptimizationMetadata, ) { - const files = await fsp.readdir(depsCacheDir) - for (const file of files) { - if (file !== 'package.json' && file !== '_metadata.json') { - const filePath = path.join(depsCacheDir, file) + for (const fileName of oldFilesNames) { + if (fileName !== 'package.json' && fileName !== '_metadata.json') { + const filePath = path.join(depsCacheDir, fileName) if (!optimizedDepInfoFromFile(metadata, normalizePath(filePath))) { fsp.unlink(filePath) } From 6b12fafcaba153a42ea2eaaccaf515893676df32 Mon Sep 17 00:00:00 2001 From: patak Date: Sun, 26 Mar 2023 12:15:46 +0200 Subject: [PATCH 4/5] chore: update --- packages/vite/src/node/optimizer/index.ts | 59 ++++++++++------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index 29bba44ca431ed..a4af5c27426cca 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -590,15 +590,24 @@ export function runOptimizeDeps( async commit() { // Write this run of pre-bundled dependencies to the deps cache - let oldFilesNames: string[] + // Get a list of old files in the deps directory to delete the stale ones + const oldFilesPaths: string[] = [] if (!fs.existsSync(depsCacheDir)) { fs.mkdirSync(depsCacheDir, { recursive: true }) - oldFilesNames = [] } else { - oldFilesNames = await fsp.readdir(depsCacheDir) + oldFilesPaths.push( + ...(await fsp.readdir(depsCacheDir)).map((f) => + path.join(depsCacheDir, f), + ), + ) } - const files = [] + const newFilesPaths = new Set() + const files: Promise[] = [] + const write = (filePath: string, content: string) => { + newFilesPaths.add(filePath) + files.push(fsp.writeFile(filePath, content)) + } // a hint for Node.js // all files in the cache directory should be recognized as ES modules @@ -606,29 +615,26 @@ export function runOptimizeDeps( depsCacheDir, 'package.json', ) - if (!fs.existsSync(packageJsonFilePath)) { - files.push( - fsp.writeFile( - packageJsonFilePath, - '{\n "type": "module"\n}\n', - ), - ) - } + if (!fs.existsSync(packageJsonFilePath)) + write(packageJsonFilePath, '{\n "type": "module"\n}\n') + else newFilesPaths.add(packageJsonFilePath) const dataPath = path.join(depsCacheDir, '_metadata.json') - files.push( - fsp.writeFile( - dataPath, - stringifyDepsOptimizerMetadata(metadata, depsCacheDir), - ), + write( + dataPath, + stringifyDepsOptimizerMetadata(metadata, depsCacheDir), ) for (const outputFile of result.outputFiles!) { - files.push(fsp.writeFile(outputFile.path, outputFile.text)) + write(outputFile.path, outputFile.text) } // Clean up old files in the background - cleanupDepsCacheStaleFiles(oldFilesNames, depsCacheDir, metadata) + for (const filePath of oldFilesPaths) { + if (!newFilesPaths.has(filePath)) { + fsp.unlink(filePath) + } + } await Promise.all(files) }, @@ -1294,18 +1300,3 @@ export async function optimizedDepNeedsInterop( } return depInfo?.needsInterop } - -async function cleanupDepsCacheStaleFiles( - oldFilesNames: string[], - depsCacheDir: string, - metadata: DepOptimizationMetadata, -) { - for (const fileName of oldFilesNames) { - if (fileName !== 'package.json' && fileName !== '_metadata.json') { - const filePath = path.join(depsCacheDir, fileName) - if (!optimizedDepInfoFromFile(metadata, normalizePath(filePath))) { - fsp.unlink(filePath) - } - } - } -} From 5ca1464bd57642943966163c988ffb444ac76869 Mon Sep 17 00:00:00 2001 From: patak Date: Sun, 26 Mar 2023 12:18:12 +0200 Subject: [PATCH 5/5] chore: simplify --- packages/vite/src/node/optimizer/index.ts | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts index a4af5c27426cca..0c0e90821bde56 100644 --- a/packages/vite/src/node/optimizer/index.ts +++ b/packages/vite/src/node/optimizer/index.ts @@ -611,30 +611,22 @@ export function runOptimizeDeps( // a hint for Node.js // all files in the cache directory should be recognized as ES modules - const packageJsonFilePath = path.resolve( - depsCacheDir, - 'package.json', + write( + path.resolve(depsCacheDir, 'package.json'), + '{\n "type": "module"\n}\n', ) - if (!fs.existsSync(packageJsonFilePath)) - write(packageJsonFilePath, '{\n "type": "module"\n}\n') - else newFilesPaths.add(packageJsonFilePath) - const dataPath = path.join(depsCacheDir, '_metadata.json') write( - dataPath, + path.join(depsCacheDir, '_metadata.json'), stringifyDepsOptimizerMetadata(metadata, depsCacheDir), ) - for (const outputFile of result.outputFiles!) { + for (const outputFile of result.outputFiles!) write(outputFile.path, outputFile.text) - } // Clean up old files in the background - for (const filePath of oldFilesPaths) { - if (!newFilesPaths.has(filePath)) { - fsp.unlink(filePath) - } - } + for (const filePath of oldFilesPaths) + if (!newFilesPaths.has(filePath)) fsp.unlink(filePath) await Promise.all(files) },