From 1f4c33fe56b3f644353ed473857b4a465a3b6ac0 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Wed, 31 Aug 2022 18:20:51 +0300 Subject: [PATCH] feat: improve performance for node loader --- packages/vite-node/src/externalize.ts | 3 +- packages/vite-node/src/types.ts | 1 + packages/vitest/src/node/config.ts | 2 +- packages/vitest/src/node/plugins/index.ts | 7 + packages/vitest/src/runtime/loader.ts | 150 ++++++++++++++++++---- 5 files changed, 134 insertions(+), 29 deletions(-) diff --git a/packages/vite-node/src/externalize.ts b/packages/vite-node/src/externalize.ts index b5331e3595e4..e0a386d94efe 100644 --- a/packages/vite-node/src/externalize.ts +++ b/packages/vite-node/src/externalize.ts @@ -81,7 +81,8 @@ async function _shouldExternalize( return id const isDist = id.includes('/dist/') - if ((isNodeModule || isDist) && await isValidNodeImport(id)) + // don't check for valid Node import, if we are using custom loader + if ((isNodeModule || isDist) && (options?.registerNodeLoader || await isValidNodeImport(id))) return id return false diff --git a/packages/vite-node/src/types.ts b/packages/vite-node/src/types.ts index b6581a641786..5082ad937322 100644 --- a/packages/vite-node/src/types.ts +++ b/packages/vite-node/src/types.ts @@ -12,6 +12,7 @@ export interface DepsHandlingOptions { * @default false */ fallbackCJS?: boolean + registerNodeLoader?: boolean } export interface StartOfSourceMap { diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 4e1b6ddba6cc..e00a19fafc1f 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -128,7 +128,7 @@ export function resolveConfig( // disable loader for Yarn PnP until Node implements chain loader // https://github.com/nodejs/node/pull/43772 - resolved.deps.registerNodeLoader ??= false + resolved.deps.registerNodeLoader ??= typeof process.versions.pnp === 'undefined' resolved.testNamePattern = resolved.testNamePattern ? resolved.testNamePattern instanceof RegExp diff --git a/packages/vitest/src/node/plugins/index.ts b/packages/vitest/src/node/plugins/index.ts index 7024634b51dc..e26c6e865ad9 100644 --- a/packages/vitest/src/node/plugins/index.ts +++ b/packages/vitest/src/node/plugins/index.ts @@ -86,6 +86,10 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest()) // setting this option can bypass that and fallback to cjs version mainFields: [], alias: preOptions.alias, + // TODO https://github.com/vitest-dev/vitest/pull/1919 + // TODO currently vue fails to import, because with new caching it assumes that whole module uses correct imports + // but exports resolves to browser esm + conditions: ['node', 'import'], }, server: { ...preOptions.api, @@ -98,6 +102,9 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest()) }, } + // if (preOptions.deps?.registerNodeLoader === false) + // config.resolve!.mainFields = [] + if (!options.browser) { // disable deps optimization Object.assign(config, { diff --git a/packages/vitest/src/runtime/loader.ts b/packages/vitest/src/runtime/loader.ts index 35103d208845..7d88fe445d83 100644 --- a/packages/vitest/src/runtime/loader.ts +++ b/packages/vitest/src/runtime/loader.ts @@ -1,10 +1,52 @@ import { pathToFileURL } from 'url' -import { readFile } from 'fs/promises' +import { readFile, writeFile } from 'fs/promises' +import { existsSync } from 'fs' +import { dirname, extname, resolve as resolvePath } from 'pathe' import { hasCJSSyntax, isNodeBuiltin } from 'mlly' import { normalizeModuleId } from 'vite-node/utils' +import { getPackageInfo } from 'local-pkg' import { getWorkerState } from '../utils' import type { Loader, ResolveResult, Resolver } from '../types/loader' import { ModuleFormat } from '../types/loader' +import type { ResolvedConfig } from '../types' + +// TODO make "optimizer"-like crawling? - check "module", "exports" fields +// TODO clear cache before running tests, if version changed + +// assume that module is bundled correctly/incorrectly alltogether +let pkgCache = new Map([ + ['/path/module', { + version: '1.0.0', + isPseudoESM: true, + }] as const, +].sort(([a], [b]) => a.length - b.length)) + +let filePkg = new Map() + +const getModuleInfo = (url: string) => { + while (url) { + const dir = dirname(url) + if (url === dir) + return null + url = dir + const cached = pkgCache.get(url) + if (cached) + return cached + } + return null +} + +const BUILTIN_EXTENSIONS = /* @__PURE__ */ new Set(['.mjs', '.cjs', '.node', '.wasm']) + +const shouldCheckForPseudoESM = (url: string) => { + if (url.startsWith('data:')) + return false + const extension = extname(url) + if (BUILTIN_EXTENSIONS.has(extension)) + return false + // skip .ts and other extensions, user should inline it + return extension === '.js' +} // TODO fix in mlly (add "}" as a possible first character: "}export default") const ESM_RE = /([\s;}]|^)(import[\w,{}\s*]*from|import\s*['"*{]|export\b\s*(?:[*{]|default|class|type|function|const|var|let|async function)|import\.meta\b)/m @@ -12,31 +54,80 @@ function hasESMSyntax(code: string) { return ESM_RE.test(code) } -interface ContextCache { - isPseudoESM: boolean - source: string +// TODO make URL relative to root +const isPseudoESM = async (url: string) => { + const shouldCheck = shouldCheckForPseudoESM(url) + if (!shouldCheck) + return false + const moduleInfo = getModuleInfo(url) + if (moduleInfo) + return moduleInfo.isPseudoESM + const pkg = await getPackageInfo(url) + if (!pkg) + return false + const pkgPath = dirname(pkg.packageJsonPath) + let isPseudoESM = false + let source: string | undefined + if (pkg.packageJson.type !== 'module') { + source = await readFile(url, 'utf8') + isPseudoESM = (hasESMSyntax(source) && !hasCJSSyntax(source)) + } + filePkg.set(url, pkgPath) + pkgCache.set(pkgPath, { + version: pkg.version, + isPseudoESM, + source, + }) + return isPseudoESM } -const cache = new Map() +let cacheRead = false +const populateCache = async (config?: ResolvedConfig) => { + const cacheDir = config?.cache && config.cache.dir + if (!cacheDir) + return // TODO recommend enabling/add option + const loaderCacheFile = resolvePath(cacheDir, 'loader-data.json') + if (!existsSync(loaderCacheFile)) { + cacheRead = true + return + } + try { + const json = await readFile(loaderCacheFile, 'utf8') + const { packages, files } = JSON.parse(json) + pkgCache = new Map(packages) + filePkg = new Map(files) + cacheRead = true + } + catch {} +} -const getPotentialSource = async (filepath: string, result: ResolveResult) => { - if (!result.url.startsWith('file://') || result.format === 'module') - return null - let source = cache.get(result.url)?.source - if (source == null) - source = await readFile(filepath, 'utf8') - return source +const saveCache = async (config?: ResolvedConfig) => { + const cacheDir = config?.cache && config.cache.dir + if (!cacheDir) + return // TODO recommend enabling/add option + const loaderCacheFile = resolvePath(cacheDir, 'loader-data.json') + const json = JSON.stringify({ + packages: Array.from(pkgCache.entries()), + files: Array.from(filePkg.entries()), + }) + try { + await writeFile(loaderCacheFile, json) + } + catch {} } -const detectESM = (url: string, source: string | null) => { - const cached = cache.get(url) - if (cached) - return cached.isPseudoESM - if (!source) - return false - return (hasESMSyntax(source) && !hasCJSSyntax(source)) +function debounce(fn: (...args: any[]) => unknown, delay: number) { + let timeoutID: NodeJS.Timeout + return function (this: unknown, ...args: unknown[]) { + globalThis.clearTimeout(timeoutID) + timeoutID = globalThis.setTimeout(() => { + fn.apply(this, args) + }, delay) + } } +const debouncedSaveCache = debounce(() => saveCache().catch(() => {}), 1000) + // apply transformations only to libraries // inline code proccessed by vite-node // make Node pseudo ESM @@ -45,6 +136,9 @@ export const resolve: Resolver = async (url, context, next) => { const state = getWorkerState() const resolver = state?.rpc.resolveId + if (!cacheRead) + await populateCache(state?.config) + if (!parentURL || isNodeBuiltin(url) || !resolver) return next(url, context, next) @@ -72,21 +166,23 @@ export const resolve: Resolver = async (url, context, next) => { } } - const source = await getPotentialSource(filepath, result) - const isPseudoESM = detectESM(result.url, source) - if (typeof source === 'string') - cache.set(result.url, { isPseudoESM, source }) - if (isPseudoESM) + const isModule = result.format !== 'module' && await isPseudoESM(filepath) + + if (isModule) result.format = ModuleFormat.Module return result } export const load: Loader = async (url, context, next) => { const result = await next(url, context, next) - const cached = cache.get(url) - if (cached?.isPseudoESM && result.format !== 'module') { + const modulePath = filePkg.get(url) + const pkgData = result.format !== 'module' && modulePath && pkgCache.get(modulePath) + if (pkgData) { + const { source } = pkgData + // TODO save before exiting process + debouncedSaveCache() return { - source: cached.source, + source: source || await readFile(url), format: ModuleFormat.Module, } }