feat: add caching to run failed and longer tests first

docs/config/index.md

@@ -565,3 +565,16 @@ RegExp pattern for files that will return en empty CSS file.
 A number of tests that are allowed to run at the same time marked with `test.concurrent`.
 
 Test above this limit will be queued to run when available slot appears.
+
+### cache
+
+- **Type**: `false | { dir? }`
+
+Options to configure Vitest cache policy. At the moment Vitest stores cache for test results to run the longer and failed tests first.
+
+#### cache.dir
+
+- **Type**: `string`
+- **Default**: `node_modules/.vitest`
+
+Path to cache directory.

docs/guide/cli.md

@@ -36,6 +36,10 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit
 vitest related /src/index.ts /src/hello-world.js
 ```
 
+### `vitest clean cache`
+
+Clears cache folder.
+
 ## Options
 
 | Options       |               |

packages/vitest/src/node/cache/files.ts

@@ -0,0 +1,23 @@
+import fs, { type Stats } from 'fs'
+
+type FileStatsCache = Pick<Stats, 'size'>
+
+export class FilesStatsCache {
+  public cache = new Map<string, FileStatsCache>()
+
+  public getStats(fsPath: string): FileStatsCache | undefined {
+    return this.cache.get(fsPath)
+  }
+
+  public async updateStats(fsPath: string) {
+    if (!fs.existsSync(fsPath))
+      return
+
+    const stats = await fs.promises.stat(fsPath)
+    this.cache.set(fsPath, { size: stats.size })
+  }
+
+  public removeStats(fsPath: string) {
+    this.cache.delete(fsPath)
+  }
+}

packages/vitest/src/node/cache/index.ts

@@ -0,0 +1,38 @@
+import fs from 'fs'
+import { findUp } from 'find-up'
+import { resolve } from 'pathe'
+import { loadConfigFromFile } from 'vite'
+import { configFiles } from '../../constants'
+import type { CliOptions } from '../cli-api'
+import { slash } from '../../utils'
+
+export class VitestCache {
+  static resolveCacheDir(root: string, dir: string | undefined) {
+    return resolve(root, slash(dir || 'node_modules/.vitest'))
+  }
+
+  static async clearCache(options: CliOptions) {
+    const root = resolve(options.root || process.cwd())
+
+    const configPath = options.config
+      ? resolve(root, options.config)
+      : await findUp(configFiles, { cwd: root } as any)
+
+    const config = await loadConfigFromFile({ command: 'serve', mode: 'test' }, configPath)
+
+    const cache = config?.config.test?.cache
+
+    if (cache === false)
+      throw new Error('Cache is disabled')
+
+    const cachePath = VitestCache.resolveCacheDir(root, cache?.dir)
+
+    let cleared = false
+
+    if (fs.existsSync(cachePath)) {
+      fs.rmSync(cachePath, { recursive: true, force: true })
+      cleared = true
+    }
+    return { dir: cachePath, cleared }
+  }
+}

packages/vitest/src/node/cache/results.ts

@@ -0,0 +1,76 @@
+import fs from 'fs'
+import { dirname, resolve } from 'pathe'
+import type { File, ResolvedConfig } from '../../types'
+import { version } from '../../../package.json'
+
+export interface SuiteResultCache {
+  failed: boolean
+  duration: number
+}
+
+export class ResultsCache {
+  private cache = new Map<string, SuiteResultCache>()
+  private cachePath: string | null = null
+  private version: string = version
+  private root = '/'
+
+  setConfig(root: string, config: ResolvedConfig['cache']) {
+    this.root = root
+    if (config)
+      this.cachePath = resolve(config.dir, 'results.json')
+  }
+
+  getResults(fsPath: string) {
+    return this.cache.get(fsPath?.slice(this.root.length))
+  }
+
+  async readFromCache() {
+    if (!this.cachePath)
+      return
+
+    if (fs.existsSync(this.cachePath)) {
+      const resultsCache = await fs.promises.readFile(this.cachePath, 'utf8')
+      const { results, version } = JSON.parse(resultsCache)
+      this.cache = new Map(results)
+      this.version = version
+    }
+  }
+
+  updateResults(files: File[]) {
+    files.forEach((file) => {
+      const result = file.result
+      if (!result)
+        return
+      const duration = result.duration || 0
+      // store as relative, so cache would be the same in CI and locally
+      const relativePath = file.filepath?.slice(this.root.length)
+      this.cache.set(relativePath, {
+        duration: duration >= 0 ? duration : 0,
+        failed: result.state === 'fail',
+      })
+    })
+  }
+
+  removeFromCache(filepath: string) {
+    this.cache.delete(filepath)
+  }
+
+  async writeToCache() {
+    if (!this.cachePath)
+      return
+
+    const results = Array.from(this.cache.entries())
+
+    const cacheDirname = dirname(this.cachePath)
+
+    if (!fs.existsSync(cacheDirname))
+      await fs.promises.mkdir(cacheDirname, { recursive: true })
+
+    const cache = JSON.stringify({
+      version: this.version,
+      results,
+    })
+
+    await fs.promises.writeFile(this.cachePath, cache)
+  }
+}

packages/vitest/src/node/config.ts

@@ -8,6 +8,7 @@ import { defaultPort } from '../constants'
 import { configDefaults } from '../defaults'
 import { resolveC8Options } from '../integrations/coverage'
 import { toArray } from '../utils'
+import { VitestCache } from './cache'
 
 const extraInlineDeps = [
   /^(?!.*(?:node_modules)).*\.mjs$/,
@@ -181,5 +182,9 @@ export function resolveConfig(
   if (typeof resolved.css === 'object')
     resolved.css.include ??= [/\.module\./]
 
+  resolved.cache ??= { dir: '' }
+  if (resolved.cache)
+    resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir)
+
   return resolved
 }

packages/vitest/src/node/core.ts

@@ -91,6 +91,9 @@ export class Vitest {
 
     if (resolved.coverage.enabled)
       await cleanCoverage(resolved.coverage, resolved.coverage.clean)
+
+    this.state.results.setConfig(resolved.root, resolved.cache)
+    await this.state.results.readFromCache()
   }
 
   getSerializableConfig() {
@@ -133,6 +136,9 @@ export class Vitest {
       process.exit(exitCode)
     }
 
+    // populate once, update cache on watch
+    await Promise.all(files.map(file => this.state.stats.updateStats(file)))
+
     await this.runFiles(files)
 
     if (this.config.coverage.enabled)
@@ -205,7 +211,7 @@ export class Vitest {
     return runningTests
   }
 
-  async runFiles(files: string[]) {
+  async runFiles(paths: string[]) {
     await this.runningPromise
 
     this.runningPromise = (async () => {
@@ -217,16 +223,21 @@ export class Vitest {
       this.snapshot.clear()
       this.state.clearErrors()
       try {
-        await this.pool.runTests(files, invalidates)
+        await this.pool.runTests(paths, invalidates)
       }
       catch (err) {
         this.state.catchError(err, 'Unhandled Error')
       }
 
-      if (hasFailed(this.state.getFiles()))
+      const files = this.state.getFiles()
+
+      if (hasFailed(files))
         process.exitCode = 1
 
-      await this.report('onFinished', this.state.getFiles(), this.state.getUnhandledErrors())
+      await this.report('onFinished', files, this.state.getUnhandledErrors())
+
+      this.state.results.updateResults(files)
+      await this.state.results.writeToCache()
     })()
       .finally(() => {
         this.runningPromise = undefined
@@ -352,6 +363,8 @@ export class Vitest {
 
       if (this.state.filesMap.has(id)) {
         this.state.filesMap.delete(id)
+        this.state.results.removeFromCache(id)
+        this.state.stats.removeStats(id)
         this.changedTests.delete(id)
         this.report('onTestRemoved', id)
       }
@@ -360,6 +373,7 @@ export class Vitest {
       id = slash(id)
       if (await this.isTargetFile(id)) {
         this.changedTests.add(id)
+        await this.state.stats.updateStats(id)
         this.scheduleRerun(id)
       }
     }

packages/vitest/src/node/pool.ts

@@ -1,16 +1,16 @@
 import { MessageChannel } from 'worker_threads'
 import { pathToFileURL } from 'url'
 import { cpus } from 'os'
-import { createHash } from 'crypto'
 import { resolve } from 'pathe'
 import type { Options as TinypoolOptions } from 'tinypool'
 import { Tinypool } from 'tinypool'
 import { createBirpc } from 'birpc'
 import type { RawSourceMap } from 'vite-node'
 import type { ResolvedConfig, WorkerContext, WorkerRPC } from '../types'
 import { distDir } from '../constants'
-import { AggregateError, slash } from '../utils'
+import { AggregateError } from '../utils'
 import type { Vitest } from './core'
+import { BaseSequelizer } from './sequelizers/BaseSequelizer'
 
 export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise<void>
 
@@ -86,29 +86,15 @@ export function createPool(ctx: Vitest): WorkerPool {
       }
     }
 
+    const sequelizer = new BaseSequelizer(ctx)
+
     return async (files, invalidates) => {
       const config = ctx.getSerializableConfig()
 
-      if (config.shard) {
-        const { index, count } = config.shard
-        const shardSize = Math.ceil(files.length / count)
-        const shardStart = shardSize * (index - 1)
-        const shardEnd = shardSize * index
-        files = files
-          .map((file) => {
-            const fullPath = resolve(slash(config.root), slash(file))
-            const specPath = fullPath.slice(config.root.length)
-            return {
-              file,
-              hash: createHash('sha1')
-                .update(specPath)
-                .digest('hex'),
-            }
-          })
-          .sort((a, b) => (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0))
-          .slice(shardStart, shardEnd)
-          .map(({ file }) => file)
-      }
+      if (config.shard)
+        files = await sequelizer.shard(files)
+
+      files = await sequelizer.sort(files)
 
       if (!ctx.config.threads) {
         await runFiles(config, files)

packages/vitest/src/node/sequelizers/BaseSequelizer.ts

@@ -0,0 +1,66 @@
+import { createHash } from 'crypto'
+import { resolve } from 'pathe'
+import { slash } from 'vite-node/utils'
+import type { Vitest } from '../core'
+import type { TestSequelizer } from './types'
+
+export class BaseSequelizer implements TestSequelizer {
+  protected ctx: Vitest
+
+  constructor(ctx: Vitest) {
+    this.ctx = ctx
+  }
+
+  // async so it can be extended by other sequelizers
+  public async shard(files: string[]): Promise<string[]> {
+    const { config } = this.ctx
+    const { index, count } = config.shard!
+    const shardSize = Math.ceil(files.length / count)
+    const shardStart = shardSize * (index - 1)
+    const shardEnd = shardSize * index
+    return [...files]
+      .map((file) => {
+        const fullPath = resolve(slash(config.root), slash(file))
+        const specPath = fullPath?.slice(config.root.length)
+        return {
+          file,
+          hash: createHash('sha1')
+            .update(specPath)
+            .digest('hex'),
+        }
+      })
+      .sort((a, b) => (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0))
+      .slice(shardStart, shardEnd)
+      .map(({ file }) => file)
+  }
+
+  // async so it can be extended by other sequelizers
+  public async sort(files: string[]): Promise<string[]> {
+    const { state } = this.ctx
+    return [...files].sort((a, b) => {
+      const aState = state.getFileTestResults(a)
+      const bState = state.getFileTestResults(b)
+
+      if (!aState || !bState) {
+        const statsA = state.getFileStats(a)
+        const statsB = state.getFileStats(b)
+
+        // run unknown first
+        if (!statsA || !statsB)
+          return !statsA && statsB ? -1 : !statsB && statsA ? 1 : 0
+
+        // run larger files first
+        return statsB.size - statsA.size
+      }
+
+      // run failed first
+      if (aState.failed && !bState.failed)
+        return -1
+      if (!aState.failed && bState.failed)
+        return 1
+
+      // run longer first
+      return bState.duration - aState.duration
+    })
+  }
+}

packages/vitest/src/node/sequelizers/types.ts

@@ -0,0 +1,15 @@
+import type { Awaitable } from '../../types'
+import type { Vitest } from '../core'
+
+export interface TestSequelizer {
+  /**
+   * Slicing tests into shards. Will be run before `sort`.
+   * Only run, if `shard` is defined.
+   */
+  shard(files: string[]): Awaitable<string[]>
+  sort(files: string[]): Awaitable<string[]>
+}
+
+export interface TestSequelizerContructor {
+  new (ctx: Vitest): TestSequelizer
+}