diff --git a/docs/advanced/runner.md b/docs/advanced/runner.md index a03b6d74ea28..16ed1a35a856 100644 --- a/docs/advanced/runner.md +++ b/docs/advanced/runner.md @@ -17,6 +17,13 @@ export interface VitestRunner { */ onCollected?(files: File[]): unknown + /** + * Called when test runner should cancel next test runs. + * Runner should listen for this method and mark tests and suites as skipped in + * "onBeforeRunSuite" and "onBeforeRunTest" when called. + */ + onCancel?(reason: CancelReason): unknown + /** * Called before running a single test. Doesn't have "result" yet. */ @@ -86,7 +93,7 @@ export interface VitestRunner { When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property. ::: warning -Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner`` and `BenchmarkRunner`). +Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner` and `BenchmarkRunner`). `ViteNodeRunner` exposes `executeId` method, which is used to import test files in a Vite-friendly environment. Meaning, it will resolve imports and transform file content at runtime so that Node can understand it. ::: diff --git a/docs/config/index.md b/docs/config/index.md index 8951feafdc7b..71ad44f8c32a 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1375,3 +1375,13 @@ Influences whether or not the `showDiff` flag should be included in the thrown A Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether. This config option affects truncating values in `test.each` titles and inside the assertion error message. + +### bail + +- **Type:** `number` +- **Default:** `0` +- **CLI**: `--bail=` + +Stop test execution when given number of tests have failed. + +By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index cdb4eb3eca02..b13e93804c91 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -92,6 +92,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim | `--no-color` | Removes colors from the console output | | `--inspect` | Enables Node.js inspector | | `--inspect-brk` | Enables Node.js inspector with break | +| `--bail ` | Stop test execution when given number of tests have failed | | `-h, --help` | Display available CLI options | ::: tip diff --git a/package.json b/package.json index 713ed041f43a..2c23bf7c34d0 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "test:run": "vitest run -r test/core", "test:all": "CI=true pnpm -r --stream run test --allowOnly", "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly", - "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads", + "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads", "typecheck": "tsc --noEmit", "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt", "ui:build": "vite build packages/ui", diff --git a/packages/browser/src/client/main.ts b/packages/browser/src/client/main.ts index 50a636cb7ca5..8e6deddc9b7b 100644 --- a/packages/browser/src/client/main.ts +++ b/packages/browser/src/client/main.ts @@ -1,7 +1,7 @@ import { createClient } from '@vitest/ws-client' // eslint-disable-next-line no-restricted-imports import type { ResolvedConfig } from 'vitest' -import type { VitestRunner } from '@vitest/runner' +import type { CancelReason, VitestRunner } from '@vitest/runner' import { createBrowserRunner } from './runner' import { importId } from './utils' import { setupConsoleLogSpy } from './logger' @@ -30,7 +30,16 @@ function getQueryPaths() { return url.searchParams.getAll('path') } -export const client = createClient(ENTRY_URL) +let setCancel = (_: CancelReason) => {} +const onCancel = new Promise((resolve) => { + setCancel = resolve +}) + +export const client = createClient(ENTRY_URL, { + handlers: { + onCancel: setCancel, + }, +}) const ws = client.ws @@ -103,6 +112,10 @@ async function runTests(paths: string[], config: ResolvedConfig) { runner = new BrowserRunner({ config, browserHashMap }) } + onCancel.then((reason) => { + runner?.onCancel?.(reason) + }) + if (!config.snapshotOptions.snapshotEnvironment) config.snapshotOptions.snapshotEnvironment = new BrowserSnapshotEnvironment() diff --git a/packages/browser/src/client/runner.ts b/packages/browser/src/client/runner.ts index 3bdc40c1f0d2..d73c2ccc6ea3 100644 --- a/packages/browser/src/client/runner.ts +++ b/packages/browser/src/client/runner.ts @@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl } async onAfterRunTest(task: Test) { - await super.onAfterRunTest?.() + await super.onAfterRunTest?.(task) task.result?.errors?.forEach((error) => { console.error(error.message) }) + + if (this.config.bail && task.result?.state === 'fail') { + const previousFailures = await rpc().getCountOfFailedTests() + const currentFailures = 1 + previousFailures + + if (currentFailures >= this.config.bail) { + rpc().onCancel('test-failure') + this.onCancel?.('test-failure') + } + } } async onAfterRunSuite() { diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index be651843674f..694d6c70045d 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -27,6 +27,8 @@ export interface VitestRunnerConstructor { new(config: VitestRunnerConfig): VitestRunner } +export type CancelReason = 'keyboard-input' | 'test-failure' | string & {} + export interface VitestRunner { /** * First thing that's getting called before actually collecting and running tests. @@ -37,6 +39,13 @@ export interface VitestRunner { */ onCollected?(files: File[]): unknown + /** + * Called when test runner should cancel next test runs. + * Runner should listen for this method and mark tests and suites as skipped in + * "onBeforeRunSuite" and "onBeforeRunTest" when called. + */ + onCancel?(reason: CancelReason): unknown + /** * Called before running a single test. Doesn't have "result" yet. */ diff --git a/packages/vitest/package.json b/packages/vitest/package.json index 2758f8e4b54e..8ebc85542dda 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -155,7 +155,7 @@ "std-env": "^3.3.2", "strip-literal": "^1.0.1", "tinybench": "^2.4.0", - "tinypool": "^0.4.0", + "tinypool": "^0.5.0", "vite": "^3.0.0 || ^4.0.0", "vite-node": "workspace:*", "why-is-node-running": "^2.2.2" diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index b3f76efc9c1d..545ff2cf00bc 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -111,16 +111,24 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit return ctx.updateSnapshot() return ctx.updateSnapshot([file.filepath]) }, + onCancel(reason) { + ctx.cancelCurrentRun(reason) + }, + getCountOfFailedTests() { + return ctx.state.getCountOfFailedTests() + }, }, { post: msg => ws.send(msg), on: fn => ws.on('message', fn), - eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected'], + eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onCancel'], serialize: stringify, deserialize: parse, }, ) + ctx.onCancel(reason => rpc.onCancel(reason)) + clients.set(ws, rpc) ws.on('close', () => { diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index 1178b66152c9..3f96018b2d82 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -1,4 +1,5 @@ import type { TransformResult } from 'vite' +import type { CancelReason } from '@vitest/runner' import type { AfterSuiteRunMeta, File, ModuleGraphData, Reporter, ResolvedConfig, SnapshotResult, TaskResultPack, UserConsoleLog } from '../types' export interface TransformResultWithSource extends TransformResult { @@ -10,6 +11,8 @@ export interface WebSocketHandlers { onTaskUpdate(packs: TaskResultPack[]): void onAfterSuiteRun(meta: AfterSuiteRunMeta): void onDone(name: string): void + onCancel(reason: CancelReason): void + getCountOfFailedTests(): number sendLog(log: UserConsoleLog): void getFiles(): File[] getPaths(): string[] @@ -28,4 +31,5 @@ export interface WebSocketHandlers { } export interface WebSocketEvents extends Pick { + onCancel(reason: CancelReason): void } diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 617b20494379..f9599ad667fe 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -45,6 +45,7 @@ cli .option('--inspect', 'Enable Node.js inspector') .option('--inspect-brk', 'Enable Node.js inspector with break') .option('--test-timeout