feat(watch): test run cancelling, feat: --bail option for cancelling test run (#3163)

Author: AriPerkkio

docs/advanced/runner.md

@@ -17,6 +17,13 @@ export interface VitestRunner {
    */
   onCollected?(files: File[]): unknown
 
+  /**
+   * Called when test runner should cancel next test runs.
+   * Runner should listen for this method and mark tests and suites as skipped in
+   * "onBeforeRunSuite" and "onBeforeRunTest" when called.
+   */
+  onCancel?(reason: CancelReason): unknown
+
   /**
    * Called before running a single test. Doesn't have "result" yet.
    */
@@ -86,7 +93,7 @@ export interface VitestRunner {
 When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property.
 
 ::: warning
-Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner`` and `BenchmarkRunner`).
+Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner` and `BenchmarkRunner`).
 
 `ViteNodeRunner` exposes `executeId` method, which is used to import test files in a Vite-friendly environment. Meaning, it will resolve imports and transform file content at runtime so that Node can understand it.
 :::

docs/config/index.md

@@ -1375,3 +1375,13 @@ Influences whether or not the `showDiff` flag should be included in the thrown A
 Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether.
 
 This config option affects truncating values in `test.each` titles and inside the assertion error message.
+
+### bail
+
+- **Type:** `number`
+- **Default:** `0`
+- **CLI**: `--bail=<value>`
+
+Stop test execution when given number of tests have failed.
+
+By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured.

docs/guide/cli.md

@@ -92,6 +92,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
 | `--no-color` | Removes colors from the console output |
 | `--inspect` | Enables Node.js inspector |
 | `--inspect-brk` | Enables Node.js inspector with break |
+| `--bail <number>` | Stop test execution when given number of tests have failed |
 | `-h, --help` | Display available CLI options |
 
 ::: tip

package.json

@@ -23,7 +23,7 @@
     "test:run": "vitest run -r test/core",
     "test:all": "CI=true pnpm -r --stream run test --allowOnly",
     "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly",
-    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
+    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
     "typecheck": "tsc --noEmit",
     "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt",
     "ui:build": "vite build packages/ui",

packages/browser/src/client/main.ts

@@ -1,7 +1,7 @@
 import { createClient } from '@vitest/ws-client'
 // eslint-disable-next-line no-restricted-imports
 import type { ResolvedConfig } from 'vitest'
-import type { VitestRunner } from '@vitest/runner'
+import type { CancelReason, VitestRunner } from '@vitest/runner'
 import { createBrowserRunner } from './runner'
 import { importId } from './utils'
 import { setupConsoleLogSpy } from './logger'
@@ -30,7 +30,16 @@ function getQueryPaths() {
   return url.searchParams.getAll('path')
 }
 
-export const client = createClient(ENTRY_URL)
+let setCancel = (_: CancelReason) => {}
+const onCancel = new Promise<CancelReason>((resolve) => {
+  setCancel = resolve
+})
+
+export const client = createClient(ENTRY_URL, {
+  handlers: {
+    onCancel: setCancel,
+  },
+})
 
 const ws = client.ws
 
@@ -103,6 +112,10 @@ async function runTests(paths: string[], config: ResolvedConfig) {
     runner = new BrowserRunner({ config, browserHashMap })
   }
 
+  onCancel.then((reason) => {
+    runner?.onCancel?.(reason)
+  })
+
   if (!config.snapshotOptions.snapshotEnvironment)
     config.snapshotOptions.snapshotEnvironment = new BrowserSnapshotEnvironment()
 

packages/browser/src/client/runner.ts

@@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl
     }
 
     async onAfterRunTest(task: Test) {
-      await super.onAfterRunTest?.()
+      await super.onAfterRunTest?.(task)
       task.result?.errors?.forEach((error) => {
         console.error(error.message)
       })
+
+      if (this.config.bail && task.result?.state === 'fail') {
+        const previousFailures = await rpc().getCountOfFailedTests()
+        const currentFailures = 1 + previousFailures
+
+        if (currentFailures >= this.config.bail) {
+          rpc().onCancel('test-failure')
+          this.onCancel?.('test-failure')
+        }
+      }
     }
 
     async onAfterRunSuite() {

packages/runner/src/types/runner.ts

@@ -27,6 +27,8 @@ export interface VitestRunnerConstructor {
   new(config: VitestRunnerConfig): VitestRunner
 }
 
+export type CancelReason = 'keyboard-input' | 'test-failure' | string & {}
+
 export interface VitestRunner {
   /**
    * First thing that's getting called before actually collecting and running tests.
@@ -37,6 +39,13 @@ export interface VitestRunner {
    */
   onCollected?(files: File[]): unknown
 
+  /**
+   * Called when test runner should cancel next test runs.
+   * Runner should listen for this method and mark tests and suites as skipped in
+   * "onBeforeRunSuite" and "onBeforeRunTest" when called.
+   */
+  onCancel?(reason: CancelReason): unknown
+
   /**
    * Called before running a single test. Doesn't have "result" yet.
    */

packages/vitest/package.json

@@ -155,7 +155,7 @@
     "std-env": "^3.3.2",
     "strip-literal": "^1.0.1",
     "tinybench": "^2.4.0",
-    "tinypool": "^0.4.0",
+    "tinypool": "^0.5.0",
     "vite": "^3.0.0 || ^4.0.0",
     "vite-node": "workspace:*",
     "why-is-node-running": "^2.2.2"

packages/vitest/src/api/setup.ts

@@ -111,16 +111,24 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit
             return ctx.updateSnapshot()
           return ctx.updateSnapshot([file.filepath])
         },
+        onCancel(reason) {
+          ctx.cancelCurrentRun(reason)
+        },
+        getCountOfFailedTests() {
+          return ctx.state.getCountOfFailedTests()
+        },
       },
       {
         post: msg => ws.send(msg),
         on: fn => ws.on('message', fn),
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onCancel'],
         serialize: stringify,
         deserialize: parse,
       },
     )
 
+    ctx.onCancel(reason => rpc.onCancel(reason))
+
     clients.set(ws, rpc)
 
     ws.on('close', () => {

packages/vitest/src/api/types.ts

@@ -1,4 +1,5 @@
 import type { TransformResult } from 'vite'
+import type { CancelReason } from '@vitest/runner'
 import type { AfterSuiteRunMeta, File, ModuleGraphData, Reporter, ResolvedConfig, SnapshotResult, TaskResultPack, UserConsoleLog } from '../types'
 
 export interface TransformResultWithSource extends TransformResult {
@@ -10,6 +11,8 @@ export interface WebSocketHandlers {
   onTaskUpdate(packs: TaskResultPack[]): void
   onAfterSuiteRun(meta: AfterSuiteRunMeta): void
   onDone(name: string): void
+  onCancel(reason: CancelReason): void
+  getCountOfFailedTests(): number
   sendLog(log: UserConsoleLog): void
   getFiles(): File[]
   getPaths(): string[]
@@ -28,4 +31,5 @@ export interface WebSocketHandlers {
 }
 
 export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected'> {
+  onCancel(reason: CancelReason): void
 }

packages/vitest/src/node/cli.ts

@@ -45,6 +45,7 @@ cli
   .option('--inspect', 'Enable Node.js inspector')
   .option('--inspect-brk', 'Enable Node.js inspector with break')
   .option('--test-timeout <time>', 'Default timeout of a test in milliseconds (default: 5000)')
+  .option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
   .help()
 
 cli

packages/vitest/src/node/core.ts

@@ -8,6 +8,7 @@ import c from 'picocolors'
 import { normalizeRequestId } from 'vite-node/utils'
 import { ViteNodeRunner } from 'vite-node/client'
 import { SnapshotManager } from '@vitest/snapshot/manager'
+import type { CancelReason } from '@vitest/runner'
 import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, UserWorkspaceConfig, VitestRunMode } from '../types'
 import { hasFailed, noop, slash, toArray } from '../utils'
 import { getCoverageProvider } from '../integrations/coverage'
@@ -46,6 +47,7 @@ export class Vitest {
   filenamePattern?: string
   runningPromise?: Promise<void>
   closingPromise?: Promise<void>
+  isCancelling = false
 
   isFirstRun = true
   restartsCount = 0
@@ -64,6 +66,7 @@ export class Vitest {
 
   private _onRestartListeners: OnServerRestartHandler[] = []
   private _onSetServer: OnServerRestartHandler[] = []
+  private _onCancelListeners: ((reason: CancelReason) => Promise<void> | void)[] = []
 
   async setServer(options: UserConfig, server: ViteDevServer, cliOptions: UserConfig) {
     this.unregisterWatcher?.()
@@ -395,13 +398,14 @@ export class Vitest {
 
   async runFiles(paths: WorkspaceSpec[]) {
     const filepaths = paths.map(([, file]) => file)
-
     this.state.collectPaths(filepaths)
 
     await this.report('onPathsCollected', filepaths)
 
     // previous run
     await this.runningPromise
+    this._onCancelListeners = []
+    this.isCancelling = false
 
     // schedule the new run
     this.runningPromise = (async () => {
@@ -437,6 +441,11 @@ export class Vitest {
     return await this.runningPromise
   }
 
+  async cancelCurrentRun(reason: CancelReason) {
+    this.isCancelling = true
+    await Promise.all(this._onCancelListeners.splice(0).map(listener => listener(reason)))
+  }
+
   async rerunFiles(files: string[] = this.state.getFilepaths(), trigger?: string) {
     if (this.filenamePattern) {
       const filteredFiles = await this.globTestFiles([this.filenamePattern])
@@ -760,4 +769,8 @@ export class Vitest {
   onAfterSetServer(fn: OnServerRestartHandler) {
     this._onSetServer.push(fn)
   }
+
+  onCancel(fn: (reason: CancelReason) => void) {
+    this._onCancelListeners.push(fn)
+  }
 }

packages/vitest/src/node/pools/browser.ts

@@ -15,6 +15,13 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
   }
 
   const runTests = async (project: WorkspaceProject, files: string[]) => {
+    ctx.state.clearFiles(project, files)
+
+    let isCancelled = false
+    project.ctx.onCancel(() => {
+      isCancelled = true
+    })
+
     const provider = project.browserProvider!
     providers.add(provider)
 
@@ -24,6 +31,11 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
     const isolate = project.config.isolate
     if (isolate) {
       for (const path of paths) {
+        if (isCancelled) {
+          ctx.state.cancelFiles(files.slice(paths.indexOf(path)), ctx.config.root)
+          break
+        }
+
         const url = new URL('/', origin)
         url.searchParams.append('path', path)
         url.searchParams.set('id', path)

packages/vitest/src/node/pools/child.ts

@@ -4,7 +4,7 @@ import { fork } from 'node:child_process'
 import { fileURLToPath, pathToFileURL } from 'node:url'
 import { createBirpc } from 'birpc'
 import { resolve } from 'pathe'
-import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest } from '../../types'
+import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest } from '../../types'
 import type { ChildContext } from '../../types/child'
 import type { PoolProcessOptions, ProcessPool, WorkspaceSpec } from '../pool'
 import { distDir } from '../../paths'
@@ -16,9 +16,10 @@ import { createMethodsRPC } from './rpc'
 const childPath = fileURLToPath(pathToFileURL(resolve(distDir, './child.js')).href)
 
 function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess): void {
-  createBirpc<{}, RuntimeRPC>(
+  const rpc = createBirpc<RunnerRPC, RuntimeRPC>(
     createMethodsRPC(project),
     {
+      eventNames: ['onCancel'],
       serialize: v8.serialize,
       deserialize: v => v8.deserialize(Buffer.from(v)),
       post(v) {
@@ -29,6 +30,8 @@ function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess)
       },
     },
   )
+
+  project.ctx.onCancel(reason => rpc.onCancel(reason))
 }
 
 function stringifyRegex(input: RegExp | string): string {

packages/vitest/src/node/pools/rpc.ts

@@ -58,5 +58,11 @@ export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC {
     onFinished(files) {
       project.report('onFinished', files, ctx.state.getUnhandledErrors())
     },
+    onCancel(reason) {
+      ctx.cancelCurrentRun(reason)
+    },
+    getCountOfFailedTests() {
+      return ctx.state.getCountOfFailedTests()
+    },
   }
 }

packages/vitest/src/node/pools/threads.ts

@@ -6,7 +6,7 @@ import { resolve } from 'pathe'
 import type { Options as TinypoolOptions } from 'tinypool'
 import Tinypool from 'tinypool'
 import { distDir } from '../../paths'
-import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest, WorkerContext } from '../../types'
+import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest, WorkerContext } from '../../types'
 import type { PoolProcessOptions, ProcessPool, RunWithFiles } from '../pool'
 import { envsOrder, groupFilesByEnv } from '../../utils/test-helpers'
 import { AggregateError, groupBy } from '../../utils/base'
@@ -20,9 +20,10 @@ function createWorkerChannel(project: WorkspaceProject) {
   const port = channel.port2
   const workerPort = channel.port1
 
-  createBirpc<{}, RuntimeRPC>(
+  const rpc = createBirpc<RunnerRPC, RuntimeRPC>(
     createMethodsRPC(project),
     {
+      eventNames: ['onCancel'],
       post(v) {
         port.postMessage(v)
       },
@@ -32,6 +33,8 @@ function createWorkerChannel(project: WorkspaceProject) {
     },
   )
 
+  project.ctx.onCancel(reason => rpc.onCancel(reason))
+
   return { workerPort, port }
 }
 
@@ -93,6 +96,11 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt
         // Worker got stuck and won't terminate - this may cause process to hang
         if (error instanceof Error && /Failed to terminate worker/.test(error.message))
           ctx.state.addProcessTimeoutCause(`Failed to terminate worker while running ${files.join(', ')}.`)
+
+        // Intentionally cancelled
+        else if (ctx.isCancelling && error instanceof Error && /The task has been cancelled/.test(error.message))
+          ctx.state.cancelFiles(files, ctx.config.root)
+
         else
           throw error
       }
@@ -106,6 +114,9 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt
     const sequencer = new Sequencer(ctx)
 
     return async (specs, invalidates) => {
+      // Cancel pending tasks from pool when possible
+      ctx.onCancel(() => pool.cancelPendingTasks())
+
       const configs = new Map<WorkspaceProject, ResolvedConfig>()
       const getConfig = (project: WorkspaceProject): ResolvedConfig => {
         if (configs.has(project))

packages/vitest/src/node/reporters/base.ts

@@ -13,6 +13,7 @@ const HELP_QUITE = `${c.dim('press ')}${c.bold('q')}${c.dim(' to quit')}`
 
 const WAIT_FOR_CHANGE_PASS = `\n${c.bold(c.inverse(c.green(' PASS ')))}${c.green(' Waiting for file changes...')}`
 const WAIT_FOR_CHANGE_FAIL = `\n${c.bold(c.inverse(c.red(' FAIL ')))}${c.red(' Tests failed. Watching for file changes...')}`
+const WAIT_FOR_CHANGE_CANCELLED = `\n${c.bold(c.inverse(c.red(' CANCELLED ')))}${c.red(' Test run cancelled. Watching for file changes...')}`
 
 const LAST_RUN_LOG_TIMEOUT = 1_500
 
@@ -49,6 +50,7 @@ export abstract class BaseReporter implements Reporter {
 
   async onFinished(files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors()) {
     this.end = performance.now()
+
     await this.reportSummary(files)
     if (errors.length) {
       if (!this.ctx.config.dangerouslyIgnoreUnhandledErrors)
@@ -102,8 +104,12 @@ export abstract class BaseReporter implements Reporter {
 
     const failed = errors.length > 0 || hasFailed(files)
     const failedSnap = hasFailedSnapshot(files)
+    const cancelled = this.ctx.isCancelling
+
     if (failed)
       this.ctx.logger.log(WAIT_FOR_CHANGE_FAIL)
+    else if (cancelled)
+      this.ctx.logger.log(WAIT_FOR_CHANGE_CANCELLED)
     else
       this.ctx.logger.log(WAIT_FOR_CHANGE_PASS)
 

packages/vitest/src/node/state.ts

@@ -1,3 +1,4 @@
+import { relative } from 'pathe'
 import type { ErrorWithDiff, File, Task, TaskResultPack, UserConsoleLog } from '../types'
 // can't import actual functions from utils, because it's incompatible with @vitest/browsers
 import type { AggregateError as AggregateErrorPonyfill } from '../utils'
@@ -125,4 +126,23 @@ export class StateManager {
       task.logs.push(log)
     }
   }
+
+  getCountOfFailedTests() {
+    return Array.from(this.idMap.values()).filter(t => t.result?.state === 'fail').length
+  }
+
+  cancelFiles(files: string[], root: string) {
+    this.collectFiles(files.map(filepath => ({
+      filepath,
+      name: relative(root, filepath),
+      id: filepath,
+      mode: 'skip',
+      type: 'suite',
+      result: {
+        state: 'skip',
+      },
+      // Cancelled files have not yet collected tests
+      tasks: [],
+    })))
+  }
 }

packages/vitest/src/node/stdin.ts

@@ -12,6 +12,7 @@ const keys = [
   ['t', 'filter by a test name regex pattern'],
   ['q', 'quit'],
 ]
+const cancelKeys = ['space', 'c', ...keys.map(key => key[0])]
 
 export function printShortcutsHelp() {
   stdout().write(
@@ -37,12 +38,14 @@ export function registerConsoleShortcuts(ctx: Vitest) {
       return
     }
 
-    // is running, ignore keypress
-    if (ctx.runningPromise)
-      return
-
     const name = key?.name
 
+    if (ctx.runningPromise) {
+      if (cancelKeys.includes(name))
+        await ctx.cancelCurrentRun('keyboard-input')
+      return
+    }
+
     // quit
     if (name === 'q')
       return ctx.exit(true)
@@ -83,8 +86,8 @@ export function registerConsoleShortcuts(ctx: Vitest) {
       message: 'Input test name pattern (RegExp)',
       initial: ctx.configOverride.testNamePattern?.source || '',
     }])
-    await ctx.changeNamePattern(filter.trim(), undefined, 'change pattern')
     on()
+    await ctx.changeNamePattern(filter.trim(), undefined, 'change pattern')
   }
 
   async function inputFilePattern() {
@@ -96,8 +99,8 @@ export function registerConsoleShortcuts(ctx: Vitest) {
       initial: latestFilename,
     }])
     latestFilename = filter.trim()
-    await ctx.changeFilenamePattern(filter.trim())
     on()
+    await ctx.changeFilenamePattern(filter.trim())
   }
 
   let rl: readline.Interface | undefined

packages/vitest/src/runtime/child.ts

@@ -1,8 +1,9 @@
 import v8 from 'node:v8'
 import { createBirpc } from 'birpc'
 import { parseRegexp } from '@vitest/utils'
+import type { CancelReason } from '@vitest/runner'
 import type { ResolvedConfig } from '../types'
-import type { RuntimeRPC } from '../types/rpc'
+import type { RunnerRPC, RuntimeRPC } from '../types/rpc'
 import type { ChildContext } from '../types/child'
 import { mockMap, moduleCache, startViteNode } from './execute'
 import { rpcDone } from './rpc'
@@ -14,6 +15,11 @@ function init(ctx: ChildContext) {
   process.env.VITEST_WORKER_ID = '1'
   process.env.VITEST_POOL_ID = '1'
 
+  let setCancel = (_reason: CancelReason) => {}
+  const onCancel = new Promise<CancelReason>((resolve) => {
+    setCancel = resolve
+  })
+
   // @ts-expect-error untyped global
   globalThis.__vitest_environment__ = config.environment
   // @ts-expect-error I know what I am doing :P
@@ -22,14 +28,17 @@ function init(ctx: ChildContext) {
     moduleCache,
     config,
     mockMap,
+    onCancel,
     durations: {
       environment: 0,
       prepare: performance.now(),
     },
-    rpc: createBirpc<RuntimeRPC>(
-      {},
+    rpc: createBirpc<RuntimeRPC, RunnerRPC>(
+      {
+        onCancel: setCancel,
+      },
       {
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit', 'onCancel'],
         serialize: v8.serialize,
         deserialize: v => v8.deserialize(Buffer.from(v)),
         post(v) {

packages/vitest/src/runtime/entry.ts

@@ -71,6 +71,20 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor):
     await originalOnAfterRun?.call(testRunner, files)
   }
 
+  const originalOnAfterRunTest = testRunner.onAfterRunTest
+  testRunner.onAfterRunTest = async (test) => {
+    if (config.bail && test.result?.state === 'fail') {
+      const previousFailures = await rpc().getCountOfFailedTests()
+      const currentFailures = 1 + previousFailures
+
+      if (currentFailures >= config.bail) {
+        rpc().onCancel('test-failure')
+        testRunner.onCancel?.('test-failure')
+      }
+    }
+    await originalOnAfterRunTest?.call(testRunner, test)
+  }
+
   return testRunner
 }
 
@@ -85,6 +99,7 @@ export async function run(files: string[], config: ResolvedConfig, environment:
     setupChaiConfig(config.chaiConfig)
 
   const runner = await getTestRunner(config, executor)
+  workerState.onCancel.then(reason => runner.onCancel?.(reason))
 
   workerState.durations.prepare = performance.now() - workerState.durations.prepare
 

packages/vitest/src/runtime/runners/test.ts

@@ -1,4 +1,4 @@
-import type { Suite, Test, TestContext, VitestRunner, VitestRunnerImportSource } from '@vitest/runner'
+import type { CancelReason, Suite, Test, TestContext, VitestRunner, VitestRunnerImportSource } from '@vitest/runner'
 import { GLOBAL_EXPECT, getState, setState } from '@vitest/expect'
 import { getSnapshotClient } from '../../integrations/snapshot/chai'
 import { vi } from '../../integrations/vi'
@@ -12,6 +12,7 @@ export class VitestTestRunner implements VitestRunner {
   private snapshotClient = getSnapshotClient()
   private workerState = getWorkerState()
   private __vitest_executor!: VitestExecutor
+  private cancelRun = false
 
   constructor(public config: ResolvedConfig) {}
 
@@ -45,9 +46,16 @@ export class VitestTestRunner implements VitestRunner {
     this.workerState.current = undefined
   }
 
+  onCancel(_reason: CancelReason) {
+    this.cancelRun = true
+  }
+
   async onBeforeRunTest(test: Test) {
     const name = getNames(test).slice(1).join(' > ')
 
+    if (this.cancelRun)
+      test.mode = 'skip'
+
     if (test.mode !== 'run') {
       this.snapshotClient.skipTestSnapshots(name)
       return
@@ -59,6 +67,11 @@ export class VitestTestRunner implements VitestRunner {
     this.workerState.current = test
   }
 
+  onBeforeRunSuite(suite: Suite) {
+    if (this.cancelRun)
+      suite.mode = 'skip'
+  }
+
   onBeforeTryTest(test: Test) {
     setState({
       assertionCalls: 0,

packages/vitest/src/runtime/worker.ts

@@ -1,7 +1,8 @@
 import { performance } from 'node:perf_hooks'
 import { createBirpc } from 'birpc'
 import { workerId as poolId } from 'tinypool'
-import type { RuntimeRPC, WorkerContext } from '../types'
+import type { CancelReason } from '@vitest/runner'
+import type { RunnerRPC, RuntimeRPC, WorkerContext } from '../types'
 import { getWorkerState } from '../utils/global'
 import { mockMap, moduleCache, startViteNode } from './execute'
 import { setupInspect } from './inspector'
@@ -17,6 +18,11 @@ function init(ctx: WorkerContext) {
   process.env.VITEST_WORKER_ID = String(workerId)
   process.env.VITEST_POOL_ID = String(poolId)
 
+  let setCancel = (_reason: CancelReason) => {}
+  const onCancel = new Promise<CancelReason>((resolve) => {
+    setCancel = resolve
+  })
+
   // @ts-expect-error untyped global
   globalThis.__vitest_environment__ = config.environment
   // @ts-expect-error I know what I am doing :P
@@ -25,14 +31,17 @@ function init(ctx: WorkerContext) {
     moduleCache,
     config,
     mockMap,
+    onCancel,
     durations: {
       environment: 0,
       prepare: performance.now(),
     },
-    rpc: createBirpc<RuntimeRPC>(
-      {},
+    rpc: createBirpc<RuntimeRPC, RunnerRPC>(
+      {
+        onCancel: setCancel,
+      },
       {
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit', 'onCancel'],
         post(v) { port.postMessage(v) },
         on(fn) { port.addListener('message', fn) },
       },

packages/vitest/src/types/config.ts

@@ -567,6 +567,11 @@ export interface InlineConfig {
    * https://github.com/chaijs/chai/blob/4.x.x/lib/chai/config.js
   */
   chaiConfig?: ChaiConfig
+
+  /**
+   * Stop test execution when given number of tests have failed.
+   */
+  bail?: number
 }
 
 export interface TypecheckConfig {

packages/vitest/src/types/rpc.ts

@@ -1,4 +1,5 @@
 import type { FetchResult, RawSourceMap, ViteNodeResolveId } from 'vite-node'
+import type { CancelReason } from '@vitest/runner'
 import type { EnvironmentOptions, ResolvedConfig, VitestEnvironment } from './config'
 import type { UserConsoleLog } from './general'
 import type { SnapshotResult } from './snapshot'
@@ -18,11 +19,17 @@ export interface RuntimeRPC {
   onCollected: (files: File[]) => void
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
   onTaskUpdate: (pack: TaskResultPack[]) => void
+  onCancel(reason: CancelReason): void
+  getCountOfFailedTests(): number
 
   snapshotSaved: (snapshot: SnapshotResult) => void
   resolveSnapshotPath: (testPath: string) => string
 }
 
+export interface RunnerRPC {
+  onCancel: (reason: CancelReason) => void
+}
+
 export interface ContextTestEnvironment {
   name: VitestEnvironment
   options: EnvironmentOptions | null

packages/vitest/src/types/worker.ts

@@ -1,5 +1,5 @@
 import type { MessagePort } from 'node:worker_threads'
-import type { Test } from '@vitest/runner'
+import type { CancelReason, Test } from '@vitest/runner'
 import type { ModuleCacheMap, ViteNodeResolveId } from 'vite-node'
 import type { BirpcReturn } from 'birpc'
 import type { MockMap } from './mocker'
@@ -24,6 +24,7 @@ export interface WorkerGlobalState {
   current?: Test
   filepath?: string
   environmentTeardownRun?: boolean
+  onCancel: Promise<CancelReason>
   moduleCache: ModuleCacheMap
   mockMap: MockMap
   durations: {

packages/ws-client/src/index.ts

@@ -3,6 +3,7 @@ import { createBirpc } from 'birpc'
 import { parse, stringify } from 'flatted'
 // eslint-disable-next-line no-restricted-imports
 import type { WebSocketEvents, WebSocketHandlers } from 'vitest'
+import type { CancelReason } from '@vitest/runner'
 import { StateManager } from '../../vitest/src/node/state'
 
 export * from '../../vitest/src/utils/tasks'
@@ -66,6 +67,9 @@ export function createClient(url: string, options: VitestClientOptions = {}) {
     onFinished(files) {
       handlers.onFinished?.(files)
     },
+    onCancel(reason: CancelReason) {
+      handlers.onCancel?.(reason)
+    },
   }
 
   const birpcHandlers: BirpcOptions<WebSocketHandlers> = {

pnpm-lock.yaml

@@ -0,0 +1,14 @@
+import { expect, test } from 'vitest'
+
+test('1 - first.test.ts - this should pass', async () => {
+  await new Promise(resolve => setTimeout(resolve, 250))
+  expect(true).toBeTruthy()
+})
+
+test('2 - first.test.ts - this should fail', () => {
+  expect(false).toBeTruthy()
+})
+
+test('3 - first.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})

test/bail/fixtures/test/first.test.ts

@@ -0,0 +1,17 @@
+import { expect, test } from 'vitest'
+
+// When using multi threads the first test will start before failing.test.ts fails
+const isThreads = process.env.THREADS === 'true'
+
+test(`1 - second.test.ts - this should ${isThreads ? 'pass' : 'be skipped'}`, async () => {
+  await new Promise(resolve => setTimeout(resolve, 1500))
+  expect(true).toBeTruthy()
+})
+
+test('2 - second.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})
+
+test('3 - second.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})

test/bail/fixtures/test/second.test.ts

@@ -0,0 +1,35 @@
+import { defineConfig } from 'vitest/config'
+import type { WorkspaceSpec } from 'vitest/node'
+
+class TestNameSequencer {
+  async sort(files: WorkspaceSpec[]): Promise<WorkspaceSpec[]> {
+    return [...files].sort(([, filenameA], [, filenameB]) => {
+      if (filenameA > filenameB)
+        return 1
+
+      if (filenameA < filenameB)
+        return -1
+
+      return 0
+    })
+  }
+
+  public async shard(files: WorkspaceSpec[]): Promise<WorkspaceSpec[]> {
+    return files
+  }
+}
+
+export default defineConfig({
+  test: {
+    reporters: 'verbose',
+    cache: false,
+    watch: false,
+    sequence: {
+      sequencer: TestNameSequencer,
+    },
+    browser: {
+      headless: true,
+      name: 'chrome',
+    },
+  },
+})

test/bail/fixtures/vitest.config.ts

@@ -0,0 +1,14 @@
+{
+  "name": "@vitest/test-bail",
+  "private": true,
+  "scripts": {
+    "test": "vitest"
+  },
+  "devDependencies": {
+    "@vitest/browser": "workspace:*",
+    "execa": "^7.0.0",
+    "vite": "latest",
+    "vitest": "workspace:*",
+    "webdriverio": "latest"
+  }
+}

test/bail/package.json

@@ -0,0 +1,44 @@
+import { expect, test } from 'vitest'
+import { execa } from 'execa'
+
+const configs: string[][] = []
+const pools = [['--threads', 'true'], ['--threads', 'false'], ['--single-thread']]
+
+if (process.platform !== 'win32')
+  pools.push(['--browser'])
+
+for (const isolate of ['true', 'false']) {
+  for (const pool of pools) {
+    configs.push([
+      '--bail',
+      '1',
+      '--isolate',
+      isolate,
+      ...pool,
+    ])
+  }
+}
+
+for (const config of configs) {
+  test(`should bail with "${config.join(' ')}"`, async () => {
+    const { exitCode, stdout } = await execa('vitest', [
+      '--no-color',
+      '--root',
+      'fixtures',
+      ...config,
+    ], {
+      reject: false,
+      env: { THREADS: config.join(' ').includes('--threads true') ? 'true' : 'false' },
+    })
+
+    expect(exitCode).toBe(1)
+    expect(stdout).toMatch('✓ test/first.test.ts > 1 - first.test.ts - this should pass')
+    expect(stdout).toMatch('× test/first.test.ts > 2 - first.test.ts - this should fail')
+
+    // Cancelled tests should not be run
+    expect(stdout).not.toMatch('test/first.test.ts > 3 - first.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 1 - second.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 2 - second.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 3 - second.test.ts - this should be skipped')
+  })
+}

test/bail/test/bail.test.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    reporters: 'verbose',
+    include: ['test/**/*.test.*'],
+
+    // For Windows CI mostly
+    testTimeout: process.env.CI ? 30_000 : 10_000,
+    chaiConfig: {
+      truncateThreshold: 999,
+    },
+  },
+})

test/bail/vitest.config.ts

@@ -1,7 +1,14 @@
-import { test } from 'vitest'
+import { rmSync, writeFileSync } from 'node:fs'
+import { afterEach, expect, test } from 'vitest'
 
 import { startWatchMode } from './utils'
 
+const cleanups: (() => void)[] = []
+
+afterEach(() => {
+  cleanups.splice(0).forEach(fn => fn())
+})
+
 test('quit watch mode', async () => {
   const vitest = await startWatchMode()
 
@@ -35,3 +42,40 @@ test('filter by test name', async () => {
   await vitest.waitForOutput('Test name pattern: /sum/')
   await vitest.waitForOutput('1 passed')
 })
+
+test('cancel test run', async () => {
+  const vitest = await startWatchMode()
+
+  const testPath = 'fixtures/cancel.test.ts'
+  const testCase = `// Dynamic test case
+import { afterAll, afterEach, test } from 'vitest'
+
+// These should be called even when test is cancelled
+afterAll(() => console.log('[cancel-test]: afterAll'))
+afterEach(() => console.log('[cancel-test]: afterEach'))
+
+test('1 - test that finishes', async () => {
+  console.log('[cancel-test]: test')
+
+  await new Promise(resolve => setTimeout(resolve, 1000))
+})
+
+test('2 - test that is cancelled', async () => {
+  console.log('[cancel-test]: should not run')
+})
+`
+
+  cleanups.push(() => rmSync(testPath))
+  writeFileSync(testPath, testCase, 'utf8')
+
+  // Test case is running, cancel it
+  await vitest.waitForOutput('[cancel-test]: test')
+  vitest.write('c')
+
+  // Test hooks should still be called
+  await vitest.waitForOutput('CANCELLED')
+  await vitest.waitForOutput('[cancel-test]: afterAll')
+  await vitest.waitForOutput('[cancel-test]: afterEach')
+
+  expect(vitest.output).not.include('[cancel-test]: should not run')
+})