feat: --bail option for cancelling test run

docs/config/index.md

@@ -1375,3 +1375,11 @@ Influences whether or not the `showDiff` flag should be included in the thrown A
 Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether.
 
 This config option affects truncating values in `test.each` titles and inside the assertion error message.
+
+### bail
+
+- **Type:** `number`
+- **Default:** `0`
+- **CLI**: `--bail=<value>`
+
+Stop test execution when given number of tests have failed.

package.json

@@ -23,7 +23,7 @@
     "test:run": "vitest run -r test/core",
     "test:all": "CI=true pnpm -r --stream run test --allowOnly",
     "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly",
-    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
+    "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
     "typecheck": "tsc --noEmit",
     "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt",
     "ui:build": "vite build packages/ui",

packages/browser/src/client/runner.ts

@@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl
     }
 
     async onAfterRunTest(task: Test) {
-      await super.onAfterRunTest?.()
+      await super.onAfterRunTest?.(task)
       task.result?.errors?.forEach((error) => {
         console.error(error.message)
       })
+
+      if (this.config.bail && task.result?.state === 'fail') {
+        const previousFailures = await rpc().getCountOfFailedTests()
+        const currentFailures = 1 + previousFailures
+
+        if (currentFailures >= this.config.bail) {
+          rpc().onCancel('test-failure')
+          this.onCancel?.('test-failure')
+        }
+      }
     }
 
     async onAfterRunSuite() {

packages/runner/src/types/runner.ts

@@ -27,7 +27,7 @@ export interface VitestRunnerConstructor {
   new(config: VitestRunnerConfig): VitestRunner
 }
 
-export type CancelReason = 'keyboard-input' | string & {}
+export type CancelReason = 'keyboard-input' | 'test-failure' | string & {}
 
 export interface VitestRunner {
   /**

packages/vitest/src/api/setup.ts

@@ -111,6 +111,12 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit
             return ctx.updateSnapshot()
           return ctx.updateSnapshot([file.filepath])
         },
+        onCancel(reason) {
+          ctx.cancelCurrentRun(reason)
+        },
+        getCountOfFailedTests() {
+          return ctx.state.getCountOfFailedTests()
+        },
       },
       {
         post: msg => ws.send(msg),

packages/vitest/src/api/types.ts

@@ -11,6 +11,8 @@ export interface WebSocketHandlers {
   onTaskUpdate(packs: TaskResultPack[]): void
   onAfterSuiteRun(meta: AfterSuiteRunMeta): void
   onDone(name: string): void
+  onCancel(reason: CancelReason): void
+  getCountOfFailedTests(): number
   sendLog(log: UserConsoleLog): void
   getFiles(): File[]
   getPaths(): string[]

packages/vitest/src/node/cli.ts

@@ -45,6 +45,7 @@ cli
   .option('--inspect', 'Enable Node.js inspector')
   .option('--inspect-brk', 'Enable Node.js inspector with break')
   .option('--test-timeout <time>', 'Default timeout of a test in milliseconds (default: 5000)')
+  .option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
   .help()
 
 cli

packages/vitest/src/node/pools/rpc.ts

@@ -58,5 +58,11 @@ export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC {
     onFinished(files) {
       project.report('onFinished', files, ctx.state.getUnhandledErrors())
     },
+    onCancel(reason) {
+      ctx.cancelCurrentRun(reason)
+    },
+    getCountOfFailedTests() {
+      return ctx.state.getCountOfFailedTests()
+    },
   }
 }

packages/vitest/src/node/reporters/base.ts

@@ -50,6 +50,7 @@ export abstract class BaseReporter implements Reporter {
 
   async onFinished(files = this.ctx.state.getFiles(), errors = this.ctx.state.getUnhandledErrors()) {
     this.end = performance.now()
+
     await this.reportSummary(files)
     if (errors.length) {
       if (!this.ctx.config.dangerouslyIgnoreUnhandledErrors)

packages/vitest/src/node/state.ts

@@ -127,14 +127,20 @@ export class StateManager {
     }
   }
 
+  getCountOfFailedTests() {
+    return Array.from(this.idMap.values()).filter(t => t.result?.state === 'fail').length
+  }
+
   cancelFiles(files: string[], root: string) {
     this.collectFiles(files.map(filepath => ({
       filepath,
       name: relative(root, filepath),
       id: filepath,
       mode: 'skip',
       type: 'suite',
-
+      result: {
+        state: 'skip',
+      },
       // Cancelled files have not yet collected tests
       tasks: [],
     })))

packages/vitest/src/runtime/child.ts

@@ -38,7 +38,7 @@ function init(ctx: ChildContext) {
         onCancel: setCancel,
       },
       {
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit', 'onCancel'],
         serialize: v8.serialize,
         deserialize: v => v8.deserialize(Buffer.from(v)),
         post(v) {

packages/vitest/src/runtime/entry.ts

@@ -71,6 +71,20 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor):
     await originalOnAfterRun?.call(testRunner, files)
   }
 
+  const originalOnAfterRunTest = testRunner.onAfterRunTest
+  testRunner.onAfterRunTest = async (test) => {
+    if (config.bail && test.result?.state === 'fail') {
+      const previousFailures = await rpc().getCountOfFailedTests()
+      const currentFailures = 1 + previousFailures
+
+      if (currentFailures >= config.bail) {
+        rpc().onCancel('test-failure')
+        testRunner.onCancel?.('test-failure')
+      }
+    }
+    await originalOnAfterRunTest?.call(testRunner, test)
+  }
+
   return testRunner
 }
 

packages/vitest/src/runtime/worker.ts

@@ -41,7 +41,7 @@ function init(ctx: WorkerContext) {
         onCancel: setCancel,
       },
       {
-        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'],
+        eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit', 'onCancel'],
         post(v) { port.postMessage(v) },
         on(fn) { port.addListener('message', fn) },
       },

packages/vitest/src/types/config.ts

@@ -567,6 +567,11 @@ export interface InlineConfig {
    * https://github.com/chaijs/chai/blob/4.x.x/lib/chai/config.js
   */
   chaiConfig?: ChaiConfig
+
+  /**
+   * Stop test execution when given number of tests have failed.
+   */
+  bail?: number
 }
 
 export interface TypecheckConfig {

packages/vitest/src/types/rpc.ts

@@ -20,6 +20,8 @@ export interface RuntimeRPC {
   onCollected: (files: File[]) => void
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void
   onTaskUpdate: (pack: TaskResultPack[]) => void
+  onCancel(reason: CancelReason): void
+  getCountOfFailedTests(): number
 
   snapshotSaved: (snapshot: SnapshotResult) => void
   resolveSnapshotPath: (testPath: string) => string

test/bail/fixtures/test/first.test.ts

@@ -0,0 +1,14 @@
+import { expect, test } from 'vitest'
+
+test('1 - first.test.ts - this should pass', async () => {
+  await new Promise(resolve => setTimeout(resolve, 250))
+  expect(true).toBeTruthy()
+})
+
+test('2 - first.test.ts - this should fail', () => {
+  expect(false).toBeTruthy()
+})
+
+test('3 - first.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})

test/bail/fixtures/test/second.test.ts

@@ -0,0 +1,17 @@
+import { expect, test } from 'vitest'
+
+// When using multi threads the first test will start before failing.test.ts fails
+const isThreads = process.env.THREADS === 'true'
+
+test(`1 - second.test.ts - this should ${isThreads ? 'pass' : 'be skipped'}`, async () => {
+  await new Promise(resolve => setTimeout(resolve, 1500))
+  expect(true).toBeTruthy()
+})
+
+test('2 - second.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})
+
+test('3 - second.test.ts - this should be skipped', () => {
+  expect(true).toBeTruthy()
+})

test/bail/fixtures/vitest.config.ts

@@ -0,0 +1,35 @@
+import { defineConfig } from 'vitest/config'
+import type { WorkspaceSpec } from 'vitest/node'
+
+class TestNameSequencer {
+  async sort(files: WorkspaceSpec[]): Promise<WorkspaceSpec[]> {
+    return [...files].sort(([, filenameA], [, filenameB]) => {
+      if (filenameA > filenameB)
+        return 1
+
+      if (filenameA < filenameB)
+        return -1
+
+      return 0
+    })
+  }
+
+  public async shard(files: WorkspaceSpec[]): Promise<WorkspaceSpec[]> {
+    return files
+  }
+}
+
+export default defineConfig({
+  test: {
+    reporters: 'verbose',
+    cache: false,
+    watch: false,
+    sequence: {
+      sequencer: TestNameSequencer,
+    },
+    browser: {
+      headless: true,
+      name: 'chrome',
+    },
+  },
+})

test/bail/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@vitest/test-bail",
+  "private": true,
+  "scripts": {
+    "test": "vitest"
+  },
+  "devDependencies": {
+    "@vitest/browser": "workspace:*",
+    "execa": "^7.0.0",
+    "vite": "latest",
+    "vitest": "workspace:*",
+    "webdriverio": "latest"
+  }
+}

test/bail/test/bail.test.ts

@@ -0,0 +1,44 @@
+import { expect, test } from 'vitest'
+import { execa } from 'execa'
+
+const configs: string[][] = []
+const pools = [['--threads', 'true'], ['--threads', 'false'], ['--single-thread']]
+
+if (process.platform !== 'win32')
+  pools.push(['--browser'])
+
+for (const isolate of ['true', 'false']) {
+  for (const pool of pools) {
+    configs.push([
+      '--bail',
+      '1',
+      '--isolate',
+      isolate,
+      ...pool,
+    ])
+  }
+}
+
+for (const config of configs) {
+  test(`should bail with "${config.join(' ')}"`, async () => {
+    const { exitCode, stdout } = await execa('vitest', [
+      '--no-color',
+      '--root',
+      'fixtures',
+      ...config,
+    ], {
+      reject: false,
+      env: { THREADS: config.join(' ').includes('--threads true') ? 'true' : 'false' },
+    })
+
+    expect(exitCode).toBe(1)
+    expect(stdout).toMatch('✓ test/first.test.ts > 1 - first.test.ts - this should pass')
+    expect(stdout).toMatch('× test/first.test.ts > 2 - first.test.ts - this should fail')
+
+    // Cancelled tests should not be run
+    expect(stdout).not.toMatch('test/first.test.ts > 3 - first.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 1 - second.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 2 - second.test.ts - this should be skipped')
+    expect(stdout).not.toMatch('test/second.test.ts > 3 - second.test.ts - this should be skipped')
+  })
+}

test/bail/vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    reporters: 'verbose',
+    include: ['test/**/*.test.*'],
+
+    // For Windows CI mostly
+    testTimeout: process.env.CI ? 30_000 : 10_000,
+    chaiConfig: {
+      truncateThreshold: 999,
+    },
+  },
+})