feat: add caching to run failed and longer tests first

docs/config/index.md

@@ -564,3 +564,16 @@ RegExp pattern for files that will return en empty CSS file.
 A number of tests that are allowed to run at the same time marked with `test.concurrent`.
 
 Test above this limit will be queued to run when available slot appears.
+
+### cache
+
+- **Type**: `false | { dir? }`
+
+Options to configure Vitest cache policy. At the moment Vitest stores cache for test results to run the longer and failed tests first.
+
+#### cache.dir
+
+- **Type**: `string`
+- **Default**: `node_modules/.vitest`
+
+Path to cache directory.

docs/guide/cli.md

@@ -36,6 +36,10 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit
 vitest related /src/index.ts /src/hello-world.js
 ```
 
+### `vitest clearCache`
+
+Clears cache folder.
+
 ## Options
 
 | Options       |               |

packages/vitest/src/node/cache/files.ts

@@ -0,0 +1,23 @@
+import fs, { type Stats } from 'fs'
+
+type FileStatsCache = Pick<Stats, 'size'>
+
+export class FilesStatsCache {
+  public cache = new Map<string, FileStatsCache>()
+
+  public getStats(fsPath: string): FileStatsCache | undefined {
+    return this.cache.get(fsPath)
+  }
+
+  public async updateStats(fsPath: string) {
+    if (!fs.existsSync(fsPath))
+      return
+
+    const stats = await fs.promises.stat(fsPath)
+    this.cache.set(fsPath, { size: stats.size })
+  }
+
+  public removeStats(fsPath: string) {
+    this.cache.delete(fsPath)
+  }
+}

packages/vitest/src/node/cache/index.ts

@@ -0,0 +1,41 @@
+import fs from 'fs'
+import { findUp } from 'find-up'
+import { resolve } from 'pathe'
+import { loadConfigFromFile } from 'vite'
+import { configFiles } from '../../constants'
+import type { CliOptions } from '../cli-api'
+import { slash } from '../../utils'
+
+export class VitestCache {
+  static resolveCacheDir(root: string, dir: string | undefined) {
+    return resolve(root, slash(dir || 'node_modules/.vitest'))
+  }
+
+  static async clearCache(options: CliOptions) {
+    const root = resolve(options.root || process.cwd())
+
+    const configPath = options.config
+      ? resolve(root, options.config)
+      : await findUp(configFiles, { cwd: root } as any)
+
+    const config = await loadConfigFromFile({ command: 'serve', mode: 'test' }, configPath)
+
+    if (!config)
+      throw new Error(`[vitest] Not able to load config from ${configPath}`)
+
+    const cache = config.config.test?.cache
+
+    if (cache === false)
+      throw new Error('[vitest] Cache is disabled')
+
+    const cachePath = VitestCache.resolveCacheDir(root, cache?.dir)
+
+    let cleared = false
+
+    if (fs.existsSync(cachePath)) {
+      fs.rmSync(cachePath, { recursive: true, force: true })
+      cleared = true
+    }
+    return { dir: cachePath, cleared }
+  }
+}

packages/vitest/src/node/cache/results.ts

@@ -0,0 +1,74 @@
+import fs from 'fs'
+import { dirname, resolve } from 'pathe'
+import type { File, ResolvedConfig } from '../../types'
+import { version } from '../../../package.json'
+
+export interface SuiteResultCache {
+  failed: boolean
+  duration: number
+}
+
+export class ResultsCache {
+  private cache = new Map<string, SuiteResultCache>()
+  private cachePath: string | null = null
+  private version: string = version
+  private root = '/'
+
+  setConfig(root: string, config: ResolvedConfig['cache']) {
+    this.root = root
+    if (config)
+      this.cachePath = resolve(config.dir, 'results.json')
+  }
+
+  getResults(fsPath: string) {
+    return this.cache.get(fsPath?.slice(this.root.length))
+  }
+
+  async readFromCache() {
+    if (!this.cachePath)
+      return
+
+    if (fs.existsSync(this.cachePath)) {
+      const resultsCache = await fs.promises.readFile(this.cachePath, 'utf8')
+      const { results, version } = JSON.parse(resultsCache)
+      this.cache = new Map(results)
+      this.version = version
+    }
+  }
+
+  updateResults(files: File[]) {
+    files.forEach((file) => {
+      const result = file.result
+      if (!result)
+        return
+      const duration = result.duration || 0
+      // store as relative, so cache would be the same in CI and locally
+      const relativePath = file.filepath?.slice(this.root.length)
+      this.cache.set(relativePath, {
+        duration: duration >= 0 ? duration : 0,
+        failed: result.state === 'fail',
+      })
+    })
+  }
+
+  removeFromCache(filepath: string) {
+    this.cache.delete(filepath)
+  }
+
+  async writeToCache() {
+    if (!this.cachePath)
+      return
+
+    const resultsCache = Array.from(this.cache.entries())
+
+    const cacheDirname = dirname(this.cachePath)
+
+    if (!fs.existsSync(cacheDirname))
+      await fs.promises.mkdir(cacheDirname, { recursive: true })
+
+    await fs.promises.writeFile(this.cachePath, JSON.stringify({
+      version: this.version,
+      results: resultsCache,
+    }))
+  }
+}

packages/vitest/src/node/cli.ts

@@ -1,6 +1,7 @@
 import cac from 'cac'
 import c from 'picocolors'
 import { version } from '../../package.json'
+import { VitestCache } from './cache'
 import type { CliOptions } from './cli-api'
 import { startVitest } from './cli-api'
 import { divider } from './reporters/renderers/utils'
@@ -57,8 +58,20 @@ cli
   .command('[...filters]')
   .action(start)
 
+cli.command('clearCache')
+  .action(clearCache)
+
 cli.parse()
 
+async function clearCache(args: CliOptions) {
+  const { dir, cleared } = await VitestCache.clearCache(args)
+
+  if (cleared)
+    console.log(c.bgGreen(' VITEST '), `Cache cleared at ${dir}`)
+  else
+    console.log(c.bgRed(' VITEST '), `No cache found at ${dir}`)
+}
+
 async function runRelated(relatedFiles: string[] | string, argv: CliOptions) {
   argv.related = relatedFiles
   argv.passWithNoTests ??= true

packages/vitest/src/node/config.ts

@@ -8,6 +8,7 @@ import { defaultPort } from '../constants'
 import { configDefaults } from '../defaults'
 import { resolveC8Options } from '../integrations/coverage'
 import { toArray } from '../utils'
+import { VitestCache } from './cache'
 
 const extraInlineDeps = [
   /^(?!.*(?:node_modules)).*\.mjs$/,
@@ -181,5 +182,9 @@ export function resolveConfig(
   if (typeof resolved.css === 'object')
     resolved.css.include ??= [/\.module\./]
 
+  resolved.cache ??= { dir: '' }
+  if (resolved.cache)
+    resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir)
+
   return resolved
 }

packages/vitest/src/node/core.ts

@@ -91,6 +91,9 @@ export class Vitest {
 
     if (resolved.coverage.enabled)
       await cleanCoverage(resolved.coverage, resolved.coverage.clean)
+
+    this.state.results.setConfig(resolved.root, resolved.cache)
+    await this.state.results.readFromCache()
   }
 
   getSerializableConfig() {
@@ -133,6 +136,9 @@ export class Vitest {
       process.exit(exitCode)
     }
 
+    // populate once, update cache on watch
+    await Promise.all(files.map(file => this.state.stats.updateStats(file)))
+
     await this.runFiles(files)
 
     if (this.config.coverage.enabled)
@@ -205,7 +211,7 @@ export class Vitest {
     return runningTests
   }
 
-  async runFiles(files: string[]) {
+  async runFiles(paths: string[]) {
     await this.runningPromise
 
     this.runningPromise = (async () => {
@@ -217,16 +223,21 @@ export class Vitest {
       this.snapshot.clear()
       this.state.clearErrors()
       try {
-        await this.pool.runTests(files, invalidates)
+        await this.pool.runTests(paths, invalidates)
       }
       catch (err) {
         this.state.catchError(err, 'Unhandled Error')
       }
 
-      if (hasFailed(this.state.getFiles()))
+      const files = this.state.getFiles()
+
+      if (hasFailed(files))
         process.exitCode = 1
 
-      await this.report('onFinished', this.state.getFiles(), this.state.getUnhandledErrors())
+      await this.report('onFinished', files, this.state.getUnhandledErrors())
+
+      this.state.results.updateResults(files)
+      await this.state.results.writeToCache()
     })()
       .finally(() => {
         this.runningPromise = undefined
@@ -352,6 +363,8 @@ export class Vitest {
 
       if (this.state.filesMap.has(id)) {
         this.state.filesMap.delete(id)
+        this.state.results.removeFromCache(id)
+        this.state.stats.removeStats(id)
         this.changedTests.delete(id)
         this.report('onTestRemoved', id)
       }
@@ -360,6 +373,7 @@ export class Vitest {
       id = slash(id)
       if (await this.isTargetFile(id)) {
         this.changedTests.add(id)
+        await this.state.stats.updateStats(id)
         this.scheduleRerun(id)
       }
     }

packages/vitest/src/node/pool.ts

@@ -1,16 +1,16 @@
 import { MessageChannel } from 'worker_threads'
 import { pathToFileURL } from 'url'
 import { cpus } from 'os'
-import { createHash } from 'crypto'
 import { resolve } from 'pathe'
 import type { Options as TinypoolOptions } from 'tinypool'
 import { Tinypool } from 'tinypool'
 import { createBirpc } from 'birpc'
 import type { RawSourceMap } from 'vite-node'
 import type { ResolvedConfig, WorkerContext, WorkerRPC } from '../types'
 import { distDir } from '../constants'
-import { AggregateError, slash } from '../utils'
+import { AggregateError } from '../utils'
 import type { Vitest } from './core'
+import { BaseSequelizer } from './sequelizers/BaseSequelizer'
 
 export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise<void>
 
@@ -86,29 +86,15 @@ export function createPool(ctx: Vitest): WorkerPool {
       }
     }
 
+    const sequelizer = new BaseSequelizer(ctx)
+
     return async (files, invalidates) => {
       const config = ctx.getSerializableConfig()
 
-      if (config.shard) {
-        const { index, count } = config.shard
-        const shardSize = Math.ceil(files.length / count)
-        const shardStart = shardSize * (index - 1)
-        const shardEnd = shardSize * index
-        files = files
-          .map((file) => {
-            const fullPath = resolve(slash(config.root), slash(file))
-            const specPath = fullPath.slice(config.root.length)
-            return {
-              file,
-              hash: createHash('sha1')
-                .update(specPath)
-                .digest('hex'),
-            }
-          })
-          .sort((a, b) => (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0))
-          .slice(shardStart, shardEnd)
-          .map(({ file }) => file)
-      }
+      if (config.shard)
+        files = await sequelizer.shard(files)
+
+      files = await sequelizer.sort(files)
 
       if (!ctx.config.threads) {
         await runFiles(config, files)