diff --git a/docs/api/index.md b/docs/api/index.md index 38b74aa68272..5da908cc8b25 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11,8 +11,20 @@ type Awaitable = T | PromiseLike type TestFunction = () => Awaitable interface TestOptions { + /** + * Will fail the test if it takes too long to execute + */ timeout?: number + /** + * Will retry the test specific number of times if it fails + */ retry?: number + /** + * Will repeat the same test several times even if it fails each time + * If you have "retry" option and it fails, it will use every retry in each cycle + * Useful for debugging random failings + */ + repeats?: number } ``` diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 4f4fec4fe3cf..9eaaa97a4e1d 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -125,55 +125,63 @@ export async function runTest(test: Test, runner: VitestRunner) { setCurrentTest(test) - const retry = test.retry || 1 - for (let retryCount = 0; retryCount < retry; retryCount++) { - let beforeEachCleanups: HookCleanupCallback[] = [] - try { - await runner.onBeforeTryTest?.(test, retryCount) + const repeats = typeof test.repeats === 'number' ? test.repeats : 1 - beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) + for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { + const retry = test.retry || 1 - test.result.retryCount = retryCount + for (let retryCount = 0; retryCount < retry; retryCount++) { + let beforeEachCleanups: HookCleanupCallback[] = [] + try { + await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount }) - if (runner.runTest) { - await runner.runTest(test) - } - else { - const fn = getFn(test) - if (!fn) - throw new Error('Test function is not found. Did you add it using `setFn`?') - await fn() - } + test.result.retryCount = retryCount + test.result.repeatCount = repeatCount - // some async expect will be added to this array, in case user forget to await theme - if (test.promises) { - const result = await Promise.allSettled(test.promises) - const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean) - if (errors.length) - throw errors - } + beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) - await runner.onAfterTryTest?.(test, retryCount) + if (runner.runTest) { + await runner.runTest(test) + } + else { + const fn = getFn(test) + if (!fn) + throw new Error('Test function is not found. Did you add it using `setFn`?') + await fn() + } - test.result.state = 'pass' - } - catch (e) { - failTask(test.result, e) - } + // some async expect will be added to this array, in case user forget to await theme + if (test.promises) { + const result = await Promise.allSettled(test.promises) + const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean) + if (errors.length) + throw errors + } - try { - await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite]) - await callCleanupHooks(beforeEachCleanups) - } - catch (e) { - failTask(test.result, e) - } + await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount }) - if (test.result.state === 'pass') - break + if (!test.repeats) + test.result.state = 'pass' + else if (test.repeats && retry === retryCount) + test.result.state = 'pass' + } + catch (e) { + failTask(test.result, e) + } - // update retry info - updateTask(test, runner) + try { + await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite]) + await callCleanupHooks(beforeEachCleanups) + } + catch (e) { + failTask(test.result, e) + } + + if (test.result.state === 'pass') + break + // update retry info + updateTask(test, runner) + } } if (test.result.state === 'fail') @@ -291,30 +299,30 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { catch (e) { failTask(suite.result, e) } - } - suite.result.duration = now() - start - - if (suite.mode === 'run') { - if (!hasTests(suite)) { - suite.result.state = 'fail' - if (!suite.result.error) { - const error = processError(new Error(`No test found in suite ${suite.name}`)) - suite.result.error = error - suite.result.errors = [error] + if (suite.mode === 'run') { + if (!hasTests(suite)) { + suite.result.state = 'fail' + if (!suite.result.error) { + const error = processError(new Error(`No test found in suite ${suite.name}`)) + suite.result.error = error + suite.result.errors = [error] + } + } + else if (hasFailed(suite)) { + suite.result.state = 'fail' + } + else { + suite.result.state = 'pass' } } - else if (hasFailed(suite)) { - suite.result.state = 'fail' - } - else { - suite.result.state = 'pass' - } - } - await runner.onAfterRunSuite?.(suite) + updateTask(suite, runner) - updateTask(suite, runner) + suite.result.duration = now() - start + + await runner.onAfterRunSuite?.(suite) + } } async function runSuiteChild(c: Task, runner: VitestRunner) { diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 45cb3b6991fe..16fd20232f02 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -65,6 +65,14 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m if (typeof options === 'number') options = { timeout: options } + // inherit repeats and retry from suite + if (typeof suiteOptions === 'object') { + options = { + ...suiteOptions, + ...options, + } + } + const test: Test = { id: '', type: 'test', @@ -73,6 +81,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m suite: undefined!, fails: this.fails, retry: options?.retry, + repeats: options?.repeats, } as Omit as Test if (this.concurrent || concurrent) @@ -124,6 +133,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } function initSuite() { + if (typeof suiteOptions === 'number') + suiteOptions = { timeout: suiteOptions } + suite = { id: '', type: 'suite', @@ -132,6 +144,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m shuffle, tasks: [], } + setHooks(suite, createSuiteHooks()) } diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 382b7eee04c1..be651843674f 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -44,7 +44,7 @@ export interface VitestRunner { /** * Called before actually running the test function. Already has "result" with "state" and "startTime". */ - onBeforeTryTest?(test: Test, retryCount: number): unknown + onBeforeTryTest?(test: Test, options: { retry: number; repeats: number }): unknown /** * Called after result and state are set. */ @@ -52,7 +52,7 @@ export interface VitestRunner { /** * Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws. */ - onAfterTryTest?(test: Test, retryCount: number): unknown + onAfterTryTest?(test: Test, options: { retry: number; repeats: number }): unknown /** * Called before running a single suite. Doesn't have "result" yet. diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 0edf16df69f9..5c548d87240d 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -16,6 +16,7 @@ export interface TaskBase { result?: TaskResult retry?: number meta?: any + repeats?: number } export interface TaskCustom extends TaskBase { @@ -35,6 +36,7 @@ export interface TaskResult { htmlError?: string hooks?: Partial> retryCount?: number + repeatCount?: number } export type TaskResultPack = [id: string, result: TaskResult | undefined] @@ -165,6 +167,12 @@ export interface TestOptions { * @default 1 */ retry?: number + /** + * How many times the test will repeat. + * + * @default 5 + */ + repeats?: number } export type TestAPI = ChainableTestAPI & { diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index a05ea5cc654a..31bd580b4ac6 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -111,6 +111,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (task.mode === 'skip' || task.mode === 'todo') suffix += ` ${c.dim(c.gray('[skipped]'))}` + if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1) + suffix += c.yellow(` (repeat x${task.result.repeatCount})`) + if (task.result?.duration != null) { if (task.result.duration > DURATION_LONG) suffix += c.yellow(` ${Math.round(task.result.duration)}${c.dim('ms')}`) diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts new file mode 100644 index 000000000000..7542ceada924 --- /dev/null +++ b/test/core/test/repeats.test.ts @@ -0,0 +1,50 @@ +import { afterAll, describe, expect, test } from 'vitest' + +const testNumbers: number[] = [] + +describe('testing it/test', () => { + const result = [1, 1, 1, 1, 1, 2, 2, 2] + + test('test 1', () => { + testNumbers.push(1) + }, { repeats: 5 }) + + test('test 2', () => { + testNumbers.push(2) + }, { repeats: 3 }) + + test.fails('test 3', () => { + testNumbers.push(3) + expect(testNumbers).toStrictEqual(result) + }, { repeats: 1 }) + + afterAll(() => { + result.push(3) + expect(testNumbers).toStrictEqual(result) + }) +}) + +const describeNumbers: number[] = [] + +describe('testing describe', () => { + test('test 1', () => { + describeNumbers.push(1) + }) +}, { repeats: 3 }) + +afterAll(() => { + expect(describeNumbers).toStrictEqual([1, 1, 1]) +}) + +const retryNumbers: number[] = [] + +describe('testing repeats with retry', () => { + const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + test('test 1', () => { + retryNumbers.push(1) + }, { repeats: 5, retry: 2 }) + + afterAll(() => { + expect(retryNumbers).toStrictEqual(result) + }) +}) diff --git a/test/reporters/tests/__snapshots__/html.test.ts.snap b/test/reporters/tests/__snapshots__/html.test.ts.snap index a0373aab6af9..c16b0a2c72ee 100644 --- a/test/reporters/tests/__snapshots__/html.test.ts.snap +++ b/test/reporters/tests/__snapshots__/html.test.ts.snap @@ -84,6 +84,7 @@ exports[`html reporter > resolves to "failing" status for test file "json-fail" "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "fail", @@ -155,6 +156,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "pass", @@ -201,6 +203,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "pass",