From 098f7e051f6393734567db76b2936ce35137fc5d Mon Sep 17 00:00:00 2001 From: AriPerkkio Date: Fri, 14 Apr 2023 14:54:49 +0300 Subject: [PATCH 1/2] feat(watch): test run cancelling --- docs/advanced/runner.md | 9 +++- packages/browser/src/client/main.ts | 17 +++++++- packages/runner/src/types/runner.ts | 9 ++++ packages/vitest/package.json | 2 +- packages/vitest/src/api/setup.ts | 4 +- packages/vitest/src/api/types.ts | 2 + packages/vitest/src/node/core.ts | 15 ++++++- packages/vitest/src/node/pools/browser.ts | 12 ++++++ packages/vitest/src/node/pools/child.ts | 7 +++- packages/vitest/src/node/pools/threads.ts | 15 ++++++- packages/vitest/src/node/reporters/base.ts | 5 +++ packages/vitest/src/node/state.ts | 14 +++++++ packages/vitest/src/node/stdin.ts | 15 ++++--- packages/vitest/src/runtime/child.ts | 15 +++++-- packages/vitest/src/runtime/entry.ts | 1 + packages/vitest/src/runtime/runners/test.ts | 15 ++++++- packages/vitest/src/runtime/worker.ts | 15 +++++-- packages/vitest/src/types/rpc.ts | 5 +++ packages/vitest/src/types/worker.ts | 3 +- packages/ws-client/src/index.ts | 4 ++ pnpm-lock.yaml | 8 ++-- test/watch/test/stdin.test.ts | 46 ++++++++++++++++++++- 22 files changed, 209 insertions(+), 29 deletions(-) diff --git a/docs/advanced/runner.md b/docs/advanced/runner.md index a03b6d74ea28..16ed1a35a856 100644 --- a/docs/advanced/runner.md +++ b/docs/advanced/runner.md @@ -17,6 +17,13 @@ export interface VitestRunner { */ onCollected?(files: File[]): unknown + /** + * Called when test runner should cancel next test runs. + * Runner should listen for this method and mark tests and suites as skipped in + * "onBeforeRunSuite" and "onBeforeRunTest" when called. + */ + onCancel?(reason: CancelReason): unknown + /** * Called before running a single test. Doesn't have "result" yet. */ @@ -86,7 +93,7 @@ export interface VitestRunner { When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property. ::: warning -Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner`` and `BenchmarkRunner`). +Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner` and `BenchmarkRunner`). `ViteNodeRunner` exposes `executeId` method, which is used to import test files in a Vite-friendly environment. Meaning, it will resolve imports and transform file content at runtime so that Node can understand it. ::: diff --git a/packages/browser/src/client/main.ts b/packages/browser/src/client/main.ts index 50a636cb7ca5..8e6deddc9b7b 100644 --- a/packages/browser/src/client/main.ts +++ b/packages/browser/src/client/main.ts @@ -1,7 +1,7 @@ import { createClient } from '@vitest/ws-client' // eslint-disable-next-line no-restricted-imports import type { ResolvedConfig } from 'vitest' -import type { VitestRunner } from '@vitest/runner' +import type { CancelReason, VitestRunner } from '@vitest/runner' import { createBrowserRunner } from './runner' import { importId } from './utils' import { setupConsoleLogSpy } from './logger' @@ -30,7 +30,16 @@ function getQueryPaths() { return url.searchParams.getAll('path') } -export const client = createClient(ENTRY_URL) +let setCancel = (_: CancelReason) => {} +const onCancel = new Promise((resolve) => { + setCancel = resolve +}) + +export const client = createClient(ENTRY_URL, { + handlers: { + onCancel: setCancel, + }, +}) const ws = client.ws @@ -103,6 +112,10 @@ async function runTests(paths: string[], config: ResolvedConfig) { runner = new BrowserRunner({ config, browserHashMap }) } + onCancel.then((reason) => { + runner?.onCancel?.(reason) + }) + if (!config.snapshotOptions.snapshotEnvironment) config.snapshotOptions.snapshotEnvironment = new BrowserSnapshotEnvironment() diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index be651843674f..e74b93dcbb36 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -27,6 +27,8 @@ export interface VitestRunnerConstructor { new(config: VitestRunnerConfig): VitestRunner } +export type CancelReason = 'keyboard-input' | string & {} + export interface VitestRunner { /** * First thing that's getting called before actually collecting and running tests. @@ -37,6 +39,13 @@ export interface VitestRunner { */ onCollected?(files: File[]): unknown + /** + * Called when test runner should cancel next test runs. + * Runner should listen for this method and mark tests and suites as skipped in + * "onBeforeRunSuite" and "onBeforeRunTest" when called. + */ + onCancel?(reason: CancelReason): unknown + /** * Called before running a single test. Doesn't have "result" yet. */ diff --git a/packages/vitest/package.json b/packages/vitest/package.json index 2758f8e4b54e..8ebc85542dda 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -155,7 +155,7 @@ "std-env": "^3.3.2", "strip-literal": "^1.0.1", "tinybench": "^2.4.0", - "tinypool": "^0.4.0", + "tinypool": "^0.5.0", "vite": "^3.0.0 || ^4.0.0", "vite-node": "workspace:*", "why-is-node-running": "^2.2.2" diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index b3f76efc9c1d..6fe9999ca4cb 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -115,12 +115,14 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit { post: msg => ws.send(msg), on: fn => ws.on('message', fn), - eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected'], + eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onCancel'], serialize: stringify, deserialize: parse, }, ) + ctx.onCancel(reason => rpc.onCancel(reason)) + clients.set(ws, rpc) ws.on('close', () => { diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index 1178b66152c9..e3b0dcac024f 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -1,4 +1,5 @@ import type { TransformResult } from 'vite' +import type { CancelReason } from '@vitest/runner' import type { AfterSuiteRunMeta, File, ModuleGraphData, Reporter, ResolvedConfig, SnapshotResult, TaskResultPack, UserConsoleLog } from '../types' export interface TransformResultWithSource extends TransformResult { @@ -28,4 +29,5 @@ export interface WebSocketHandlers { } export interface WebSocketEvents extends Pick { + onCancel(reason: CancelReason): void } diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index f838fd585d08..925f22ecdfcd 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -8,6 +8,7 @@ import c from 'picocolors' import { normalizeRequestId } from 'vite-node/utils' import { ViteNodeRunner } from 'vite-node/client' import { SnapshotManager } from '@vitest/snapshot/manager' +import type { CancelReason } from '@vitest/runner' import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, UserWorkspaceConfig, VitestRunMode } from '../types' import { hasFailed, noop, slash, toArray } from '../utils' import { getCoverageProvider } from '../integrations/coverage' @@ -46,6 +47,7 @@ export class Vitest { filenamePattern?: string runningPromise?: Promise closingPromise?: Promise + isCancelling = false isFirstRun = true restartsCount = 0 @@ -64,6 +66,7 @@ export class Vitest { private _onRestartListeners: OnServerRestartHandler[] = [] private _onSetServer: OnServerRestartHandler[] = [] + private _onCancelListeners: ((reason: CancelReason) => Promise | void)[] = [] async setServer(options: UserConfig, server: ViteDevServer, cliOptions: UserConfig) { this.unregisterWatcher?.() @@ -395,13 +398,14 @@ export class Vitest { async runFiles(paths: WorkspaceSpec[]) { const filepaths = paths.map(([, file]) => file) - this.state.collectPaths(filepaths) await this.report('onPathsCollected', filepaths) // previous run await this.runningPromise + this._onCancelListeners = [] + this.isCancelling = false // schedule the new run this.runningPromise = (async () => { @@ -437,6 +441,11 @@ export class Vitest { return await this.runningPromise } + async cancelCurrentRun(reason: CancelReason) { + this.isCancelling = true + await Promise.all(this._onCancelListeners.splice(0).map(listener => listener(reason))) + } + async rerunFiles(files: string[] = this.state.getFilepaths(), trigger?: string) { if (this.filenamePattern) { const filteredFiles = await this.globTestFiles([this.filenamePattern]) @@ -760,4 +769,8 @@ export class Vitest { onAfterSetServer(fn: OnServerRestartHandler) { this._onSetServer.push(fn) } + + onCancel(fn: (reason: CancelReason) => void) { + this._onCancelListeners.push(fn) + } } diff --git a/packages/vitest/src/node/pools/browser.ts b/packages/vitest/src/node/pools/browser.ts index fcdef6b87d8f..3fa8cc76a6e1 100644 --- a/packages/vitest/src/node/pools/browser.ts +++ b/packages/vitest/src/node/pools/browser.ts @@ -15,6 +15,13 @@ export function createBrowserPool(ctx: Vitest): ProcessPool { } const runTests = async (project: WorkspaceProject, files: string[]) => { + ctx.state.clearFiles(project, files) + + let isCancelled = false + project.ctx.onCancel(() => { + isCancelled = true + }) + const provider = project.browserProvider! providers.add(provider) @@ -24,6 +31,11 @@ export function createBrowserPool(ctx: Vitest): ProcessPool { const isolate = project.config.isolate if (isolate) { for (const path of paths) { + if (isCancelled) { + ctx.state.cancelFiles(files.slice(paths.indexOf(path)), ctx.config.root) + break + } + const url = new URL('/', origin) url.searchParams.append('path', path) url.searchParams.set('id', path) diff --git a/packages/vitest/src/node/pools/child.ts b/packages/vitest/src/node/pools/child.ts index 33fac5f414c3..20ba9e7b6824 100644 --- a/packages/vitest/src/node/pools/child.ts +++ b/packages/vitest/src/node/pools/child.ts @@ -4,7 +4,7 @@ import { fork } from 'node:child_process' import { fileURLToPath, pathToFileURL } from 'node:url' import { createBirpc } from 'birpc' import { resolve } from 'pathe' -import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest } from '../../types' +import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest } from '../../types' import type { ChildContext } from '../../types/child' import type { PoolProcessOptions, ProcessPool, WorkspaceSpec } from '../pool' import { distDir } from '../../paths' @@ -16,9 +16,10 @@ import { createMethodsRPC } from './rpc' const childPath = fileURLToPath(pathToFileURL(resolve(distDir, './child.js')).href) function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess): void { - createBirpc<{}, RuntimeRPC>( + const rpc = createBirpc( createMethodsRPC(project), { + eventNames: ['onCancel'], serialize: v8.serialize, deserialize: v => v8.deserialize(Buffer.from(v)), post(v) { @@ -29,6 +30,8 @@ function setupChildProcessChannel(project: WorkspaceProject, fork: ChildProcess) }, }, ) + + project.ctx.onCancel(reason => rpc.onCancel(reason)) } function stringifyRegex(input: RegExp | string): string { diff --git a/packages/vitest/src/node/pools/threads.ts b/packages/vitest/src/node/pools/threads.ts index 3335ed477788..cd1b293d4bc4 100644 --- a/packages/vitest/src/node/pools/threads.ts +++ b/packages/vitest/src/node/pools/threads.ts @@ -6,7 +6,7 @@ import { resolve } from 'pathe' import type { Options as TinypoolOptions } from 'tinypool' import Tinypool from 'tinypool' import { distDir } from '../../paths' -import type { ContextTestEnvironment, ResolvedConfig, RuntimeRPC, Vitest, WorkerContext } from '../../types' +import type { ContextTestEnvironment, ResolvedConfig, RunnerRPC, RuntimeRPC, Vitest, WorkerContext } from '../../types' import type { PoolProcessOptions, ProcessPool, RunWithFiles } from '../pool' import { envsOrder, groupFilesByEnv } from '../../utils/test-helpers' import { AggregateError, groupBy } from '../../utils/base' @@ -20,9 +20,10 @@ function createWorkerChannel(project: WorkspaceProject) { const port = channel.port2 const workerPort = channel.port1 - createBirpc<{}, RuntimeRPC>( + const rpc = createBirpc( createMethodsRPC(project), { + eventNames: ['onCancel'], post(v) { port.postMessage(v) }, @@ -32,6 +33,8 @@ function createWorkerChannel(project: WorkspaceProject) { }, ) + project.ctx.onCancel(reason => rpc.onCancel(reason)) + return { workerPort, port } } @@ -93,6 +96,11 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt // Worker got stuck and won't terminate - this may cause process to hang if (error instanceof Error && /Failed to terminate worker/.test(error.message)) ctx.state.addProcessTimeoutCause(`Failed to terminate worker while running ${files.join(', ')}.`) + + // Intentionally cancelled + else if (ctx.isCancelling && error instanceof Error && /The task has been cancelled/.test(error.message)) + ctx.state.cancelFiles(files, ctx.config.root) + else throw error } @@ -106,6 +114,9 @@ export function createThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessOpt const sequencer = new Sequencer(ctx) return async (specs, invalidates) => { + // Cancel pending tasks from pool when possible + ctx.onCancel(() => pool.cancelPendingTasks()) + const configs = new Map() const getConfig = (project: WorkspaceProject): ResolvedConfig => { if (configs.has(project)) diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 15087eec1229..484840208bec 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -13,6 +13,7 @@ const HELP_QUITE = `${c.dim('press ')}${c.bold('q')}${c.dim(' to quit')}` const WAIT_FOR_CHANGE_PASS = `\n${c.bold(c.inverse(c.green(' PASS ')))}${c.green(' Waiting for file changes...')}` const WAIT_FOR_CHANGE_FAIL = `\n${c.bold(c.inverse(c.red(' FAIL ')))}${c.red(' Tests failed. Watching for file changes...')}` +const WAIT_FOR_CHANGE_CANCELLED = `\n${c.bold(c.inverse(c.red(' CANCELLED ')))}${c.red(' Test run cancelled. Watching for file changes...')}` const LAST_RUN_LOG_TIMEOUT = 1_500 @@ -102,8 +103,12 @@ export abstract class BaseReporter implements Reporter { const failed = errors.length > 0 || hasFailed(files) const failedSnap = hasFailedSnapshot(files) + const cancelled = this.ctx.isCancelling + if (failed) this.ctx.logger.log(WAIT_FOR_CHANGE_FAIL) + else if (cancelled) + this.ctx.logger.log(WAIT_FOR_CHANGE_CANCELLED) else this.ctx.logger.log(WAIT_FOR_CHANGE_PASS) diff --git a/packages/vitest/src/node/state.ts b/packages/vitest/src/node/state.ts index b956317262dd..bbed9381447d 100644 --- a/packages/vitest/src/node/state.ts +++ b/packages/vitest/src/node/state.ts @@ -1,3 +1,4 @@ +import { relative } from 'pathe' import type { ErrorWithDiff, File, Task, TaskResultPack, UserConsoleLog } from '../types' // can't import actual functions from utils, because it's incompatible with @vitest/browsers import type { AggregateError as AggregateErrorPonyfill } from '../utils' @@ -125,4 +126,17 @@ export class StateManager { task.logs.push(log) } } + + cancelFiles(files: string[], root: string) { + this.collectFiles(files.map(filepath => ({ + filepath, + name: relative(root, filepath), + id: filepath, + mode: 'skip', + type: 'suite', + + // Cancelled files have not yet collected tests + tasks: [], + }))) + } } diff --git a/packages/vitest/src/node/stdin.ts b/packages/vitest/src/node/stdin.ts index dfcc5d4824e4..1f81ded56378 100644 --- a/packages/vitest/src/node/stdin.ts +++ b/packages/vitest/src/node/stdin.ts @@ -12,6 +12,7 @@ const keys = [ ['t', 'filter by a test name regex pattern'], ['q', 'quit'], ] +const cancelKeys = ['space', 'c', ...keys.map(key => key[0])] export function printShortcutsHelp() { stdout().write( @@ -37,12 +38,14 @@ export function registerConsoleShortcuts(ctx: Vitest) { return } - // is running, ignore keypress - if (ctx.runningPromise) - return - const name = key?.name + if (ctx.runningPromise) { + if (cancelKeys.includes(name)) + await ctx.cancelCurrentRun('keyboard-input') + return + } + // quit if (name === 'q') return ctx.exit(true) @@ -83,8 +86,8 @@ export function registerConsoleShortcuts(ctx: Vitest) { message: 'Input test name pattern (RegExp)', initial: ctx.configOverride.testNamePattern?.source || '', }]) - await ctx.changeNamePattern(filter.trim(), undefined, 'change pattern') on() + await ctx.changeNamePattern(filter.trim(), undefined, 'change pattern') } async function inputFilePattern() { @@ -96,8 +99,8 @@ export function registerConsoleShortcuts(ctx: Vitest) { initial: latestFilename, }]) latestFilename = filter.trim() - await ctx.changeFilenamePattern(filter.trim()) on() + await ctx.changeFilenamePattern(filter.trim()) } let rl: readline.Interface | undefined diff --git a/packages/vitest/src/runtime/child.ts b/packages/vitest/src/runtime/child.ts index 29a6d690333b..9240b20226e7 100644 --- a/packages/vitest/src/runtime/child.ts +++ b/packages/vitest/src/runtime/child.ts @@ -1,8 +1,9 @@ import v8 from 'node:v8' import { createBirpc } from 'birpc' import { parseRegexp } from '@vitest/utils' +import type { CancelReason } from '@vitest/runner' import type { ResolvedConfig } from '../types' -import type { RuntimeRPC } from '../types/rpc' +import type { RunnerRPC, RuntimeRPC } from '../types/rpc' import type { ChildContext } from '../types/child' import { mockMap, moduleCache, startViteNode } from './execute' import { rpcDone } from './rpc' @@ -14,6 +15,11 @@ function init(ctx: ChildContext) { process.env.VITEST_WORKER_ID = '1' process.env.VITEST_POOL_ID = '1' + let setCancel = (_reason: CancelReason) => {} + const onCancel = new Promise((resolve) => { + setCancel = resolve + }) + // @ts-expect-error untyped global globalThis.__vitest_environment__ = config.environment // @ts-expect-error I know what I am doing :P @@ -22,12 +28,15 @@ function init(ctx: ChildContext) { moduleCache, config, mockMap, + onCancel, durations: { environment: 0, prepare: performance.now(), }, - rpc: createBirpc( - {}, + rpc: createBirpc( + { + onCancel: setCancel, + }, { eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'], serialize: v8.serialize, diff --git a/packages/vitest/src/runtime/entry.ts b/packages/vitest/src/runtime/entry.ts index 471f02fd6497..5c42e497ab16 100644 --- a/packages/vitest/src/runtime/entry.ts +++ b/packages/vitest/src/runtime/entry.ts @@ -85,6 +85,7 @@ export async function run(files: string[], config: ResolvedConfig, environment: setupChaiConfig(config.chaiConfig) const runner = await getTestRunner(config, executor) + workerState.onCancel.then(reason => runner.onCancel?.(reason)) workerState.durations.prepare = performance.now() - workerState.durations.prepare diff --git a/packages/vitest/src/runtime/runners/test.ts b/packages/vitest/src/runtime/runners/test.ts index 1abcfce709ce..3e2eebfd1220 100644 --- a/packages/vitest/src/runtime/runners/test.ts +++ b/packages/vitest/src/runtime/runners/test.ts @@ -1,4 +1,4 @@ -import type { Suite, Test, TestContext, VitestRunner, VitestRunnerImportSource } from '@vitest/runner' +import type { CancelReason, Suite, Test, TestContext, VitestRunner, VitestRunnerImportSource } from '@vitest/runner' import { GLOBAL_EXPECT, getState, setState } from '@vitest/expect' import { getSnapshotClient } from '../../integrations/snapshot/chai' import { vi } from '../../integrations/vi' @@ -12,6 +12,7 @@ export class VitestTestRunner implements VitestRunner { private snapshotClient = getSnapshotClient() private workerState = getWorkerState() private __vitest_executor!: VitestExecutor + private cancelRun = false constructor(public config: ResolvedConfig) {} @@ -45,9 +46,16 @@ export class VitestTestRunner implements VitestRunner { this.workerState.current = undefined } + onCancel(_reason: CancelReason) { + this.cancelRun = true + } + async onBeforeRunTest(test: Test) { const name = getNames(test).slice(1).join(' > ') + if (this.cancelRun) + test.mode = 'skip' + if (test.mode !== 'run') { this.snapshotClient.skipTestSnapshots(name) return @@ -59,6 +67,11 @@ export class VitestTestRunner implements VitestRunner { this.workerState.current = test } + onBeforeRunSuite(suite: Suite) { + if (this.cancelRun) + suite.mode = 'skip' + } + onBeforeTryTest(test: Test) { setState({ assertionCalls: 0, diff --git a/packages/vitest/src/runtime/worker.ts b/packages/vitest/src/runtime/worker.ts index 105d470f08f3..983303333931 100644 --- a/packages/vitest/src/runtime/worker.ts +++ b/packages/vitest/src/runtime/worker.ts @@ -1,7 +1,8 @@ import { performance } from 'node:perf_hooks' import { createBirpc } from 'birpc' import { workerId as poolId } from 'tinypool' -import type { RuntimeRPC, WorkerContext } from '../types' +import type { CancelReason } from '@vitest/runner' +import type { RunnerRPC, RuntimeRPC, WorkerContext } from '../types' import { getWorkerState } from '../utils/global' import { mockMap, moduleCache, startViteNode } from './execute' import { setupInspect } from './inspector' @@ -17,6 +18,11 @@ function init(ctx: WorkerContext) { process.env.VITEST_WORKER_ID = String(workerId) process.env.VITEST_POOL_ID = String(poolId) + let setCancel = (_reason: CancelReason) => {} + const onCancel = new Promise((resolve) => { + setCancel = resolve + }) + // @ts-expect-error untyped global globalThis.__vitest_environment__ = config.environment // @ts-expect-error I know what I am doing :P @@ -25,12 +31,15 @@ function init(ctx: WorkerContext) { moduleCache, config, mockMap, + onCancel, durations: { environment: 0, prepare: performance.now(), }, - rpc: createBirpc( - {}, + rpc: createBirpc( + { + onCancel: setCancel, + }, { eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onWorkerExit'], post(v) { port.postMessage(v) }, diff --git a/packages/vitest/src/types/rpc.ts b/packages/vitest/src/types/rpc.ts index 294b64147051..7099faaf3953 100644 --- a/packages/vitest/src/types/rpc.ts +++ b/packages/vitest/src/types/rpc.ts @@ -1,4 +1,5 @@ import type { FetchResult, RawSourceMap, ViteNodeResolveId } from 'vite-node' +import type { CancelReason } from '@vitest/runner' import type { EnvironmentOptions, ResolvedConfig, VitestEnvironment } from './config' import type { UserConsoleLog } from './general' import type { SnapshotResult } from './snapshot' @@ -23,6 +24,10 @@ export interface RuntimeRPC { resolveSnapshotPath: (testPath: string) => string } +export interface RunnerRPC { + onCancel: (reason: CancelReason) => void +} + export interface ContextTestEnvironment { name: VitestEnvironment options: EnvironmentOptions | null diff --git a/packages/vitest/src/types/worker.ts b/packages/vitest/src/types/worker.ts index cc0face52dbe..b6224e8174fe 100644 --- a/packages/vitest/src/types/worker.ts +++ b/packages/vitest/src/types/worker.ts @@ -1,5 +1,5 @@ import type { MessagePort } from 'node:worker_threads' -import type { Test } from '@vitest/runner' +import type { CancelReason, Test } from '@vitest/runner' import type { ModuleCacheMap, ViteNodeResolveId } from 'vite-node' import type { BirpcReturn } from 'birpc' import type { MockMap } from './mocker' @@ -24,6 +24,7 @@ export interface WorkerGlobalState { current?: Test filepath?: string environmentTeardownRun?: boolean + onCancel: Promise moduleCache: ModuleCacheMap mockMap: MockMap durations: { diff --git a/packages/ws-client/src/index.ts b/packages/ws-client/src/index.ts index d016e12c8cee..097e9198ced4 100644 --- a/packages/ws-client/src/index.ts +++ b/packages/ws-client/src/index.ts @@ -3,6 +3,7 @@ import { createBirpc } from 'birpc' import { parse, stringify } from 'flatted' // eslint-disable-next-line no-restricted-imports import type { WebSocketEvents, WebSocketHandlers } from 'vitest' +import type { CancelReason } from '@vitest/runner' import { StateManager } from '../../vitest/src/node/state' export * from '../../vitest/src/utils/tasks' @@ -66,6 +67,9 @@ export function createClient(url: string, options: VitestClientOptions = {}) { onFinished(files) { handlers.onFinished?.(files) }, + onCancel(reason: CancelReason) { + handlers.onCancel?.(reason) + }, } const birpcHandlers: BirpcOptions = { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dccbf739883b..4a7170a41214 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1222,8 +1222,8 @@ importers: specifier: ^2.4.0 version: 2.4.0 tinypool: - specifier: ^0.4.0 - version: 0.4.0 + specifier: ^0.5.0 + version: 0.5.0 vite: specifier: ^4.2.1 version: 4.2.1(@types/node@18.7.13) @@ -22629,8 +22629,8 @@ packages: resolution: {integrity: sha512-iyziEiyFxX4kyxSp+MtY1oCH/lvjH3PxFN8PGCDeqcZWAJ/i+9y+nL85w99PxVzrIvew/GSkSbDYtiGVa85Afg==} dev: false - /tinypool@0.4.0: - resolution: {integrity: sha512-2ksntHOKf893wSAH4z/+JbPpi92esw8Gn9N2deXX+B0EO92hexAVI9GIZZPx7P5aYo5KULfeOSt3kMOmSOy6uA==} + /tinypool@0.5.0: + resolution: {integrity: sha512-paHQtnrlS1QZYKF/GnLoOM/DN9fqaGOFbCbxzAhwniySnzl9Ebk8w73/dd34DAhe/obUbPAOldTyYXQZxnPBPQ==} engines: {node: '>=14.0.0'} dev: false diff --git a/test/watch/test/stdin.test.ts b/test/watch/test/stdin.test.ts index f39e8c41d234..fd4d13a2af00 100644 --- a/test/watch/test/stdin.test.ts +++ b/test/watch/test/stdin.test.ts @@ -1,7 +1,14 @@ -import { test } from 'vitest' +import { rmSync, writeFileSync } from 'node:fs' +import { afterEach, expect, test } from 'vitest' import { startWatchMode } from './utils' +const cleanups: (() => void)[] = [] + +afterEach(() => { + cleanups.splice(0).forEach(fn => fn()) +}) + test('quit watch mode', async () => { const vitest = await startWatchMode() @@ -35,3 +42,40 @@ test('filter by test name', async () => { await vitest.waitForOutput('Test name pattern: /sum/') await vitest.waitForOutput('1 passed') }) + +test('cancel test run', async () => { + const vitest = await startWatchMode() + + const testPath = 'fixtures/cancel.test.ts' + const testCase = `// Dynamic test case +import { afterAll, afterEach, test } from 'vitest' + +// These should be called even when test is cancelled +afterAll(() => console.log('[cancel-test]: afterAll')) +afterEach(() => console.log('[cancel-test]: afterEach')) + +test('1 - test that finishes', async () => { + console.log('[cancel-test]: test') + + await new Promise(resolve => setTimeout(resolve, 1000)) +}) + +test('2 - test that is cancelled', async () => { + console.log('[cancel-test]: should not run') +}) +` + + cleanups.push(() => rmSync(testPath)) + writeFileSync(testPath, testCase, 'utf8') + + // Test case is running, cancel it + await vitest.waitForOutput('[cancel-test]: test') + vitest.write('c') + + // Test hooks should still be called + await vitest.waitForOutput('CANCELLED') + await vitest.waitForOutput('[cancel-test]: afterAll') + await vitest.waitForOutput('[cancel-test]: afterEach') + + expect(vitest.output).not.include('[cancel-test]: should not run') +}) From d228836cb3798c3289b461184a62f816fc578408 Mon Sep 17 00:00:00 2001 From: AriPerkkio Date: Tue, 18 Apr 2023 15:20:41 +0300 Subject: [PATCH 2/2] feat: `--bail` option for cancelling test run --- docs/config/index.md | 10 +++++ docs/guide/cli.md | 1 + package.json | 2 +- packages/browser/src/client/runner.ts | 12 +++++- packages/runner/src/types/runner.ts | 2 +- packages/vitest/src/api/setup.ts | 6 +++ packages/vitest/src/api/types.ts | 2 + packages/vitest/src/node/cli.ts | 1 + packages/vitest/src/node/pools/rpc.ts | 6 +++ packages/vitest/src/node/reporters/base.ts | 1 + packages/vitest/src/node/state.ts | 8 +++- packages/vitest/src/runtime/child.ts | 2 +- packages/vitest/src/runtime/entry.ts | 14 +++++++ packages/vitest/src/runtime/worker.ts | 2 +- packages/vitest/src/types/config.ts | 5 +++ packages/vitest/src/types/rpc.ts | 2 + pnpm-lock.yaml | 18 +++++++++ test/bail/fixtures/test/first.test.ts | 14 +++++++ test/bail/fixtures/test/second.test.ts | 17 +++++++++ test/bail/fixtures/vitest.config.ts | 35 +++++++++++++++++ test/bail/package.json | 14 +++++++ test/bail/test/bail.test.ts | 44 ++++++++++++++++++++++ test/bail/vitest.config.ts | 14 +++++++ 23 files changed, 226 insertions(+), 6 deletions(-) create mode 100644 test/bail/fixtures/test/first.test.ts create mode 100644 test/bail/fixtures/test/second.test.ts create mode 100644 test/bail/fixtures/vitest.config.ts create mode 100644 test/bail/package.json create mode 100644 test/bail/test/bail.test.ts create mode 100644 test/bail/vitest.config.ts diff --git a/docs/config/index.md b/docs/config/index.md index 8951feafdc7b..71ad44f8c32a 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1375,3 +1375,13 @@ Influences whether or not the `showDiff` flag should be included in the thrown A Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether. This config option affects truncating values in `test.each` titles and inside the assertion error message. + +### bail + +- **Type:** `number` +- **Default:** `0` +- **CLI**: `--bail=` + +Stop test execution when given number of tests have failed. + +By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index cdb4eb3eca02..b13e93804c91 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -92,6 +92,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim | `--no-color` | Removes colors from the console output | | `--inspect` | Enables Node.js inspector | | `--inspect-brk` | Enables Node.js inspector with break | +| `--bail ` | Stop test execution when given number of tests have failed | | `-h, --help` | Display available CLI options | ::: tip diff --git a/package.json b/package.json index 713ed041f43a..2c23bf7c34d0 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "test:run": "vitest run -r test/core", "test:all": "CI=true pnpm -r --stream run test --allowOnly", "test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly", - "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads", + "test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads", "typecheck": "tsc --noEmit", "typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt", "ui:build": "vite build packages/ui", diff --git a/packages/browser/src/client/runner.ts b/packages/browser/src/client/runner.ts index 3bdc40c1f0d2..d73c2ccc6ea3 100644 --- a/packages/browser/src/client/runner.ts +++ b/packages/browser/src/client/runner.ts @@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl } async onAfterRunTest(task: Test) { - await super.onAfterRunTest?.() + await super.onAfterRunTest?.(task) task.result?.errors?.forEach((error) => { console.error(error.message) }) + + if (this.config.bail && task.result?.state === 'fail') { + const previousFailures = await rpc().getCountOfFailedTests() + const currentFailures = 1 + previousFailures + + if (currentFailures >= this.config.bail) { + rpc().onCancel('test-failure') + this.onCancel?.('test-failure') + } + } } async onAfterRunSuite() { diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index e74b93dcbb36..694d6c70045d 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -27,7 +27,7 @@ export interface VitestRunnerConstructor { new(config: VitestRunnerConfig): VitestRunner } -export type CancelReason = 'keyboard-input' | string & {} +export type CancelReason = 'keyboard-input' | 'test-failure' | string & {} export interface VitestRunner { /** diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index 6fe9999ca4cb..545ff2cf00bc 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -111,6 +111,12 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit return ctx.updateSnapshot() return ctx.updateSnapshot([file.filepath]) }, + onCancel(reason) { + ctx.cancelCurrentRun(reason) + }, + getCountOfFailedTests() { + return ctx.state.getCountOfFailedTests() + }, }, { post: msg => ws.send(msg), diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index e3b0dcac024f..3f96018b2d82 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -11,6 +11,8 @@ export interface WebSocketHandlers { onTaskUpdate(packs: TaskResultPack[]): void onAfterSuiteRun(meta: AfterSuiteRunMeta): void onDone(name: string): void + onCancel(reason: CancelReason): void + getCountOfFailedTests(): number sendLog(log: UserConsoleLog): void getFiles(): File[] getPaths(): string[] diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 617b20494379..f9599ad667fe 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -45,6 +45,7 @@ cli .option('--inspect', 'Enable Node.js inspector') .option('--inspect-brk', 'Enable Node.js inspector with break') .option('--test-timeout