feat(watch): test run cancelling, feat: --bail option for cancellin…

…g test run (#3163)

docs/advanced/runner.md

@@ -17,6 +17,13 @@ export interface VitestRunner {
    */
   onCollected?(files: File[]): unknown
 
+  /**
+   * Called when test runner should cancel next test runs.
+   * Runner should listen for this method and mark tests and suites as skipped in
+   * "onBeforeRunSuite" and "onBeforeRunTest" when called.
+   */
+  onCancel?(reason: CancelReason): unknown
+
   /**
    * Called before running a single test. Doesn't have "result" yet.
    */
@@ -86,7 +93,7 @@ export interface VitestRunner {
 When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property.
 
 ::: warning
-Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner`` and `BenchmarkRunner`).
+Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner` and `BenchmarkRunner`).
 
 `ViteNodeRunner` exposes `executeId` method, which is used to import test files in a Vite-friendly environment. Meaning, it will resolve imports and transform file content at runtime so that Node can understand it.
 :::

docs/config/index.md

@@ -1375,3 +1375,13 @@ Influences whether or not the `showDiff` flag should be included in the thrown A
 Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether.
 
 This config option affects truncating values in `test.each` titles and inside the assertion error message.
+
+### bail
+
+- **Type:** `number`
+- **Default:** `0`
+- **CLI**: `--bail=<value>`
+
+Stop test execution when given number of tests have failed.
+
+By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured.

docs/guide/cli.md

@@ -92,6 +92,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
 | `--no-color` | Removes colors from the console output |
 | `--inspect` | Enables Node.js inspector |
 | `--inspect-brk` | Enables Node.js inspector with break |
+| `--bail <number>` | Stop test execution when given number of tests have failed |
 | `-h, --help` | Display available CLI options |
 
 ::: tip

package.json

@@ -23,7 +23,7 @@
     "test:run": "vitest run -r test/core",
     "test:all": "CI=true pnpm -r --stream run test --allowOnly",
     "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly",
-    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
+    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
     "typecheck": "tsc --noEmit",
     "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt",
     "ui:build": "vite build packages/ui",

packages/browser/src/client/main.ts

@@ -1,7 +1,7 @@
 import { createClient } from '@vitest/ws-client'
 // eslint-disable-next-line no-restricted-imports
 import type { ResolvedConfig } from 'vitest'
-import type { VitestRunner } from '@vitest/runner'
+import type { CancelReason, VitestRunner } from '@vitest/runner'
 import { createBrowserRunner } from './runner'
 import { importId } from './utils'
 import { setupConsoleLogSpy } from './logger'
@@ -30,7 +30,16 @@ function getQueryPaths() {
   return url.searchParams.getAll('path')
 }
 
-export const client = createClient(ENTRY_URL)
+let setCancel = (_: CancelReason) => {}
+const onCancel = new Promise<CancelReason>((resolve) => {
+  setCancel = resolve
+})
+
+export const client = createClient(ENTRY_URL, {
+  handlers: {
+    onCancel: setCancel,
+  },
+})
 
 const ws = client.ws
 
@@ -103,6 +112,10 @@ async function runTests(paths: string[], config: ResolvedConfig) {
     runner = new BrowserRunner({ config, browserHashMap })
   }
 
+  onCancel.then((reason) => {
+    runner?.onCancel?.(reason)
+  })
+
   if (!config.snapshotOptions.snapshotEnvironment)
     config.snapshotOptions.snapshotEnvironment = new BrowserSnapshotEnvironment()
 

packages/browser/src/client/runner.ts

@@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl
     }
 
     async onAfterRunTest(task: Test) {
-      await super.onAfterRunTest?.()
+      await super.onAfterRunTest?.(task)
       task.result?.errors?.forEach((error) => {
         console.error(error.message)
       })
+
+      if (this.config.bail && task.result?.state === 'fail') {
+        const previousFailures = await rpc().getCountOfFailedTests()
+        const currentFailures = 1 + previousFailures
+
+        if (currentFailures >= this.config.bail) {
+          rpc().onCancel('test-failure')
+          this.onCancel?.('test-failure')
+        }
+      }
     }
 
     async onAfterRunSuite() {

packages/runner/src/types/runner.ts

@@ -27,6 +27,8 @@ export interface VitestRunnerConstructor {
   new(config: VitestRunnerConfig): VitestRunner
 }
 
+export type CancelReason = 'keyboard-input' | 'test-failure' | string & {}
+
 export interface VitestRunner {
   /**
    * First thing that's getting called before actually collecting and running tests.
@@ -37,6 +39,13 @@ export interface VitestRunner {
    */
   onCollected?(files: File[]): unknown
 
+  /**
+   * Called when test runner should cancel next test runs.
+   * Runner should listen for this method and mark tests and suites as skipped in
+   * "onBeforeRunSuite" and "onBeforeRunTest" when called.
+   */
+  onCancel?(reason: CancelReason): unknown
+
   /**
    * Called before running a single test. Doesn't have "result" yet.
    */

packages/vitest/package.json

@@ -155,7 +155,7 @@
     "std-env": "^3.3.2",
     "strip-literal": "^1.0.1",
     "tinybench": "^2.4.0",
-    "tinypool": "^0.4.0",
+    "tinypool": "^0.5.0",
     "vite": "^3.0.0 || ^4.0.0",
     "vite-node": "workspace:*",
     "why-is-node-running": "^2.2.2"

packages/vitest/src/api/setup.ts

@@ -111,16 +111,24 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit
             return ctx.updateSnapshot()
           return ctx.updateSnapshot([file.filepath])
         },
+        onCancel(reason) {
+          ctx.cancelCurrentRun(reason)
+        },
+        getCountOfFailedTests() {
+          return ctx.state.getCountOfFailedTests()
+        },
       },
       {
         post: msg => ws.send(msg),
         on: fn => ws.on('message', fn),
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onCancel'],
         serialize: stringify,
         deserialize: parse,
       },
     )
 
+    ctx.onCancel(reason => rpc.onCancel(reason))
+
     clients.set(ws, rpc)
 
     ws.on('close', () => {

packages/vitest/src/api/types.ts

@@ -1,4 +1,5 @@
 import type { TransformResult } from 'vite'
+import type { CancelReason } from '@vitest/runner'
 import type { AfterSuiteRunMeta, File, ModuleGraphData, Reporter, ResolvedConfig, SnapshotResult, TaskResultPack, UserConsoleLog } from '../types'
 
 export interface TransformResultWithSource extends TransformResult {
@@ -10,6 +11,8 @@ export interface WebSocketHandlers {
   onTaskUpdate(packs: TaskResultPack[]): void
   onAfterSuiteRun(meta: AfterSuiteRunMeta): void
   onDone(name: string): void
+  onCancel(reason: CancelReason): void
+  getCountOfFailedTests(): number
   sendLog(log: UserConsoleLog): void
   getFiles(): File[]
   getPaths(): string[]
@@ -28,4 +31,5 @@ export interface WebSocketHandlers {
 }
 
 export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected'> {
+  onCancel(reason: CancelReason): void
 }

packages/vitest/src/node/cli.ts

@@ -45,6 +45,7 @@ cli
   .option('--inspect', 'Enable Node.js inspector')
   .option('--inspect-brk', 'Enable Node.js inspector with break')
   .option('--test-timeout <time>', 'Default timeout of a test in milliseconds (default: 5000)')
+  .option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
   .help()
 
 cli

packages/vitest/src/node/core.ts

@@ -8,6 +8,7 @@ import c from 'picocolors'
 import { normalizeRequestId } from 'vite-node/utils'
 import { ViteNodeRunner } from 'vite-node/client'
 import { SnapshotManager } from '@vitest/snapshot/manager'
+import type { CancelReason } from '@vitest/runner'
 import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, UserWorkspaceConfig, VitestRunMode } from '../types'
 import { hasFailed, noop, slash, toArray } from '../utils'
 import { getCoverageProvider } from '../integrations/coverage'
@@ -46,6 +47,7 @@ export class Vitest {
   filenamePattern?: string
   runningPromise?: Promise<void>
   closingPromise?: Promise<void>
+  isCancelling = false
 
   isFirstRun = true
   restartsCount = 0
@@ -64,6 +66,7 @@ export class Vitest {
 
   private _onRestartListeners: OnServerRestartHandler[] = []
   private _onSetServer: OnServerRestartHandler[] = []
+  private _onCancelListeners: ((reason: CancelReason) => Promise<void> | void)[] = []
 
   async setServer(options: UserConfig, server: ViteDevServer, cliOptions: UserConfig) {
     this.unregisterWatcher?.()
@@ -395,13 +398,14 @@ export class Vitest {
 
   async runFiles(paths: WorkspaceSpec[]) {
     const filepaths = paths.map(([, file]) => file)
-
     this.state.collectPaths(filepaths)
 
     await this.report('onPathsCollected', filepaths)
 
     // previous run
     await this.runningPromise
+    this._onCancelListeners = []
+    this.isCancelling = false
 
     // schedule the new run
     this.runningPromise = (async () => {
@@ -437,6 +441,11 @@ export class Vitest {
     return await this.runningPromise
   }
 
+  async cancelCurrentRun(reason: CancelReason) {
+    this.isCancelling = true
+    await Promise.all(this._onCancelListeners.splice(0).map(listener => listener(reason)))
+  }
+
   async rerunFiles(files: string[] = this.state.getFilepaths(), trigger?: string) {
     if (this.filenamePattern) {
       const filteredFiles = await this.globTestFiles([this.filenamePattern])
@@ -760,4 +769,8 @@ export class Vitest {
   onAfterSetServer(fn: OnServerRestartHandler) {
     this._onSetServer.push(fn)
   }
+
+  onCancel(fn: (reason: CancelReason) => void) {
+    this._onCancelListeners.push(fn)
+  }
 }

packages/vitest/src/node/pools/browser.ts

@@ -15,6 +15,13 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
   }
 
   const runTests = async (project: WorkspaceProject, files: string[]) => {
+    ctx.state.clearFiles(project, files)
+
+    let isCancelled = false
+    project.ctx.onCancel(() => {
+      isCancelled = true
+    })
+
     const provider = project.browserProvider!
     providers.add(provider)
 
@@ -24,6 +31,11 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
     const isolate = project.config.isolate
     if (isolate) {
       for (const path of paths) {
+        if (isCancelled) {
+          ctx.state.cancelFiles(files.slice(paths.indexOf(path)), ctx.config.root)
+          break
+        }
+
         const url = new URL('/', origin)
         url.searchParams.append('path', path)
         url.searchParams.set('id', path)

packages/vitest/src/node/pools/child.ts

@@ -4,7 +4,7 @@ import { fork } from 'node:child_process'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import { createBirpc } from 'birpc'
 import { resolve } from 'pathe'
-import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest } from '../../types'
+import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest } from '../../types'
 import type { ChildContext } from '../../types/child'
 import type { PoolProcessOptions, ProcessPool, WorkspaceSpec } from '../pool'
 import { distDir } from '../../paths'
@@ -16,9 +16,10 @@ import { createMethodsRPC } from './rpc'
 const childPath = fileURLToPath(pathToFileURL(resolve(distDir, './child.js')).href)
 
 function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess): void {
-  createBirpc<{}, RuntimeRPC>(
+  const rpc = createBirpc<RunnerRPC, RuntimeRPC>(
     createMethodsRPC(project),
     {
+      eventNames: ['onCancel'],
       serialize: v8.serialize,
       deserialize: v => v8.deserialize(Buffer.from(v)),
       post(v) {
@@ -29,6 +30,8 @@ function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess)
       },
     },
   )
+
+  project.ctx.onCancel(reason => rpc.onCancel(reason))
 }
 
 function stringifyRegex(input: RegExp | string): string {

packages/vitest/src/node/pools/rpc.ts

@@ -58,5 +58,11 @@ export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC {
     onFinished(files) {
       project.report('onFinished', files, ctx.state.getUnhandledErrors())
     },
+    onCancel(reason) {
+      ctx.cancelCurrentRun(reason)
+    },
+    getCountOfFailedTests() {
+      return ctx.state.getCountOfFailedTests()
+    },
   }
 }

packages/vitest/src/node/pools/threads.ts

@@ -6,7 +6,7 @@ import { resolve } from 'pathe'
 import type { Options as TinypoolOptions } from 'tinypool'
 import Tinypool from 'tinypool'
 import { distDir } from '../../paths'
-import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest, WorkerContext } from '../../types'
+import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest, WorkerContext } from '../../types'
 import type { PoolProcessOptions, ProcessPool, RunWithFiles } from '../pool'
 import { envsOrder, groupFilesByEnv } from '../../utils/test-helpers'
 import { AggregateError, groupBy } from '../../utils/base'
@@ -20,9 +20,10 @@ function createWorkerChannel(project: WorkspaceProject) {
   const port = channel.port2
   const workerPort = channel.port1
 
-  createBirpc<{}, RuntimeRPC>(
+  const rpc = createBirpc<RunnerRPC, RuntimeRPC>(
     createMethodsRPC(project),
     {
+      eventNames: ['onCancel'],
       post(v) {
         port.postMessage(v)
       },
@@ -32,6 +33,8 @@ function createWorkerChannel(project: WorkspaceProject) {
     },
   )
 
+  project.ctx.onCancel(reason => rpc.onCancel(reason))
+
   return { workerPort, port }
 }
 
@@ -93,6 +96,11 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt
         // Worker got stuck and won't terminate - this may cause process to hang
         if (error instanceof Error && /Failed to terminate worker/.test(error.message))
           ctx.state.addProcessTimeoutCause(`Failed to terminate worker while running ${files.join(', ')}.`)
+
+        // Intentionally cancelled
+        else if (ctx.isCancelling && error instanceof Error && /The task has been cancelled/.test(error.message))
+          ctx.state.cancelFiles(files, ctx.config.root)
+
         else
           throw error
       }
@@ -106,6 +114,9 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt
     const sequencer = new Sequencer(ctx)
 
     return async (specs, invalidates) => {
+      // Cancel pending tasks from pool when possible
+      ctx.onCancel(() => pool.cancelPendingTasks())
+
       const configs = new Map<WorkspaceProject, ResolvedConfig>()
       const getConfig = (project: WorkspaceProject): ResolvedConfig => {
         if (configs.has(project))