feat: improve performance for node loader

packages/vite-node/src/externalize.ts

@@ -81,7 +81,8 @@ async function _shouldExternalize(
     return id
 
   const isDist = id.includes('/dist/')
-  if ((isNodeModule || isDist) && await isValidNodeImport(id))
+  // don't check for valid Node import, if we are using custom loader
+  if ((isNodeModule || isDist) && (options?.registerNodeLoader || await isValidNodeImport(id)))
     return id
 
   return false

packages/vite-node/src/types.ts

@@ -12,6 +12,7 @@ export interface DepsHandlingOptions {
    * @default false
    */
   fallbackCJS?: boolean
+  registerNodeLoader?: boolean
 }
 
 export interface StartOfSourceMap {

packages/vitest/src/node/config.ts

@@ -128,7 +128,7 @@ export function resolveConfig(
 
   // disable loader for Yarn PnP until Node implements chain loader
   // https://github.com/nodejs/node/pull/43772
-  resolved.deps.registerNodeLoader ??= false
+  resolved.deps.registerNodeLoader ??= typeof process.versions.pnp === 'undefined'
 
   resolved.testNamePattern = resolved.testNamePattern
     ? resolved.testNamePattern instanceof RegExp

packages/vitest/src/node/plugins/index.ts

@@ -86,6 +86,10 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest())
             // setting this option can bypass that and fallback to cjs version
             mainFields: [],
             alias: preOptions.alias,
+            // TODO https://github.com/vitest-dev/vitest/pull/1919
+            // TODO currently vue fails to import, because with new caching it assumes that whole module uses correct imports
+            // but exports resolves to browser esm
+            conditions: ['node', 'import'],
           },
           server: {
             ...preOptions.api,
@@ -98,6 +102,9 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest())
           },
         }
 
+        // if (preOptions.deps?.registerNodeLoader === false)
+        //   config.resolve!.mainFields = []
+
         if (!options.browser) {
           // disable deps optimization
           Object.assign(config, {

packages/vitest/src/runtime/loader.ts

@@ -1,42 +1,133 @@
 import { pathToFileURL } from 'url'
-import { readFile } from 'fs/promises'
+import { readFile, writeFile } from 'fs/promises'
+import { existsSync } from 'fs'
+import { dirname, extname, resolve as resolvePath } from 'pathe'
 import { hasCJSSyntax, isNodeBuiltin } from 'mlly'
 import { normalizeModuleId } from 'vite-node/utils'
+import { getPackageInfo } from 'local-pkg'
 import { getWorkerState } from '../utils'
 import type { Loader, ResolveResult, Resolver } from '../types/loader'
 import { ModuleFormat } from '../types/loader'
+import type { ResolvedConfig } from '../types'
+
+// TODO make "optimizer"-like crawling? - check "module", "exports" fields
+// TODO clear cache before running tests, if version changed
+
+// assume that module is bundled correctly/incorrectly alltogether
+let pkgCache = new Map<string, { isPseudoESM: boolean; version: string; source?: string }>([
+  ['/path/module', {
+    version: '1.0.0',
+    isPseudoESM: true,
+  }] as const,
+].sort(([a], [b]) => a.length - b.length))
+
+let filePkg = new Map<string, string>()
+
+const getModuleInfo = (url: string) => {
+  while (url) {
+    const dir = dirname(url)
+    if (url === dir)
+      return null
+    url = dir
+    const cached = pkgCache.get(url)
+    if (cached)
+      return cached
+  }
+  return null
+}
+
+const BUILTIN_EXTENSIONS = /* @__PURE__ */ new Set(['.mjs', '.cjs', '.node', '.wasm'])
+
+const shouldCheckForPseudoESM = (url: string) => {
+  if (url.startsWith('data:'))
+    return false
+  const extension = extname(url)
+  if (BUILTIN_EXTENSIONS.has(extension))
+    return false
+  // skip .ts and other extensions, user should inline it
+  return extension === '.js'
+}
 
 // TODO fix in mlly (add "}" as a possible first character: "}export default")
 const ESM_RE = /([\s;}]|^)(import[\w,{}\s*]*from|import\s*['"*{]|export\b\s*(?:[*{]|default|class|type|function|const|var|let|async function)|import\.meta\b)/m
 function hasESMSyntax(code: string) {
   return ESM_RE.test(code)
 }
 
-interface ContextCache {
-  isPseudoESM: boolean
-  source: string
+// TODO make URL relative to root
+const isPseudoESM = async (url: string) => {
+  const shouldCheck = shouldCheckForPseudoESM(url)
+  if (!shouldCheck)
+    return false
+  const moduleInfo = getModuleInfo(url)
+  if (moduleInfo)
+    return moduleInfo.isPseudoESM
+  const pkg = await getPackageInfo(url)
+  if (!pkg)
+    return false
+  const pkgPath = dirname(pkg.packageJsonPath)
+  let isPseudoESM = false
+  let source: string | undefined
+  if (pkg.packageJson.type !== 'module') {
+    source = await readFile(url, 'utf8')
+    isPseudoESM = (hasESMSyntax(source) && !hasCJSSyntax(source))
+  }
+  filePkg.set(url, pkgPath)
+  pkgCache.set(pkgPath, {
+    version: pkg.version,
+    isPseudoESM,
+    source,
+  })
+  return isPseudoESM
 }
 
-const cache = new Map<string, ContextCache>()
+let cacheRead = false
+const populateCache = async (config?: ResolvedConfig) => {
+  const cacheDir = config?.cache && config.cache.dir
+  if (!cacheDir)
+    return // TODO recommend enabling/add option
+  const loaderCacheFile = resolvePath(cacheDir, 'loader-data.json')
+  if (!existsSync(loaderCacheFile)) {
+    cacheRead = true
+    return
+  }
+  try {
+    const json = await readFile(loaderCacheFile, 'utf8')
+    const { packages, files } = JSON.parse(json)
+    pkgCache = new Map(packages)
+    filePkg = new Map(files)
+    cacheRead = true
+  }
+  catch {}
+}
 
-const getPotentialSource = async (filepath: string, result: ResolveResult) => {
-  if (!result.url.startsWith('file://') || result.format === 'module')
-    return null
-  let source = cache.get(result.url)?.source
-  if (source == null)
-    source = await readFile(filepath, 'utf8')
-  return source
+const saveCache = async (config?: ResolvedConfig) => {
+  const cacheDir = config?.cache && config.cache.dir
+  if (!cacheDir)
+    return // TODO recommend enabling/add option
+  const loaderCacheFile = resolvePath(cacheDir, 'loader-data.json')
+  const json = JSON.stringify({
+    packages: Array.from(pkgCache.entries()),
+    files: Array.from(filePkg.entries()),
+  })
+  try {
+    await writeFile(loaderCacheFile, json)
+  }
+  catch {}
 }
 
-const detectESM = (url: string, source: string | null) => {
-  const cached = cache.get(url)
-  if (cached)
-    return cached.isPseudoESM
-  if (!source)
-    return false
-  return (hasESMSyntax(source) && !hasCJSSyntax(source))
+function debounce(fn: (...args: any[]) => unknown, delay: number) {
+  let timeoutID: NodeJS.Timeout
+  return function (this: unknown, ...args: unknown[]) {
+    globalThis.clearTimeout(timeoutID)
+    timeoutID = globalThis.setTimeout(() => {
+      fn.apply(this, args)
+    }, delay)
+  }
 }
 
+const debouncedSaveCache = debounce(() => saveCache().catch(() => {}), 1000)
+
 // apply transformations only to libraries
 // inline code proccessed by vite-node
 // make Node pseudo ESM
@@ -45,6 +136,9 @@ export const resolve: Resolver = async (url, context, next) => {
   const state = getWorkerState()
   const resolver = state?.rpc.resolveId
 
+  if (!cacheRead)
+    await populateCache(state?.config)
+
   if (!parentURL || isNodeBuiltin(url) || !resolver)
     return next(url, context, next)
 
@@ -72,21 +166,23 @@ export const resolve: Resolver = async (url, context, next) => {
     }
   }
 
-  const source = await getPotentialSource(filepath, result)
-  const isPseudoESM = detectESM(result.url, source)
-  if (typeof source === 'string')
-    cache.set(result.url, { isPseudoESM, source })
-  if (isPseudoESM)
+  const isModule = result.format !== 'module' && await isPseudoESM(filepath)
+
+  if (isModule)
     result.format = ModuleFormat.Module
   return result
 }
 
 export const load: Loader = async (url, context, next) => {
   const result = await next(url, context, next)
-  const cached = cache.get(url)
-  if (cached?.isPseudoESM && result.format !== 'module') {
+  const modulePath = filePkg.get(url)
+  const pkgData = result.format !== 'module' && modulePath && pkgCache.get(modulePath)
+  if (pkgData) {
+    const { source } = pkgData
+    // TODO save before exiting process
+    debouncedSaveCache()
     return {
-      source: cached.source,
+      source: source || await readFile(url),
       format: ModuleFormat.Module,
     }
   }