From a5cd00cae28e30604100b54c6132f2573fe50030 Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 12 Jan 2023 16:00:06 -0300 Subject: [PATCH 1/9] Repeat method to run a test multiple time --- packages/runner/src/suite.ts | 12 +++++++----- packages/runner/src/types/tasks.ts | 6 ++++-- packages/vitest/src/node/reporters/json.ts | 3 ++- .../vitest/src/node/reporters/renderers/utils.ts | 2 ++ packages/vitest/src/typecheck/collect.ts | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 11a25d13c7f9..a1547c274d7a 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -60,7 +60,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m initSuite() const test = createTest(function (name: string, fn = noop, options = suiteOptions) { - const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' + const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' if (typeof options === 'number') options = { timeout: options } @@ -73,6 +73,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m suite: undefined!, fails: this.fails, retry: options?.retry, + // 5 repetitions by default + repeats: mode === 'repeats' && !options?.repeats ? 5 : options?.repeats, } as Omit as Test if (this.concurrent || concurrent) @@ -169,7 +171,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { function suiteFn(this: Record, name: string, factory?: SuiteFactory, options?: number | TestOptions) { - const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' + const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle, options) } @@ -194,14 +196,14 @@ function createSuite() { suiteFn.runIf = (condition: any) => (condition ? suite : suite.skip) as SuiteAPI return createChainable( - ['concurrent', 'shuffle', 'skip', 'only', 'todo'], + ['concurrent', 'shuffle', 'skip', 'only', 'todo', 'repeats'], suiteFn, ) as unknown as SuiteAPI } function createTest(fn: ( ( - this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>, + this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails' | 'repeats', boolean | undefined>, title: string, fn?: TestFunction, options?: number | TestOptions @@ -231,7 +233,7 @@ function createTest(fn: ( testFn.runIf = (condition: any) => (condition ? test : test.skip) as TestAPI return createChainable( - ['concurrent', 'skip', 'only', 'todo', 'fails'], + ['concurrent', 'skip', 'only', 'todo', 'fails', 'repeats'], testFn, ) as TestAPI } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index c917deb5739e..7c306e6d1e7e 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -2,7 +2,7 @@ import type { Awaitable } from '@vitest/utils' import type { ChainableFunction } from '../utils/chain' import type { ErrorWithDiff } from '../utils/error' -export type RunMode = 'run' | 'skip' | 'only' | 'todo' +export type RunMode = 'run' | 'skip' | 'only' | 'todo' | 'repeats' export type TaskState = RunMode | 'pass' | 'fail' export interface TaskBase { @@ -140,7 +140,7 @@ interface TestEachFunction { } type ChainableTestAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'fails', + 'concurrent' | 'only' | 'skip' | 'todo' | 'fails' | 'repeats', [name: string, fn?: TestFunction, options?: number | TestOptions], void, { @@ -161,6 +161,8 @@ export interface TestOptions { * @default 1 */ retry?: number + + repeats?: number } export type TestAPI = ChainableTestAPI & { diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 533255288736..3b70383fefa3 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -10,7 +10,7 @@ import { parseErrorStacktrace } from '../../utils/source-map' // the following types are extracted from the Jest repository (and simplified) // the commented-out fields are the missing ones -type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' +type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' | 'repeated' type Milliseconds = number interface Callsite { line: number; column: number } const StatusMap: Record = { @@ -20,6 +20,7 @@ const StatusMap: Record = { run: 'pending', skip: 'skipped', todo: 'todo', + repeats: 'repeated', } interface FormattedAssertionResult { diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index bb31e14ec173..37adb0ea22a4 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -103,11 +103,13 @@ export function getStateString(tasks: Task[], name = 'tests', showTotal = true) const failed = tasks.filter(i => i.result?.state === 'fail') const skipped = tasks.filter(i => i.mode === 'skip') const todo = tasks.filter(i => i.mode === 'todo') + const repeated = tasks.filter(i => i.mode === 'repeats') return [ failed.length ? c.bold(c.red(`${failed.length} failed`)) : null, passed.length ? c.bold(c.green(`${passed.length} passed`)) : null, skipped.length ? c.yellow(`${skipped.length} skipped`) : null, + repeated.length ? c.yellow(`${repeated.length} repeated`) : null, todo.length ? c.gray(`${todo.length} todo`) : null, ].filter(Boolean).join(c.dim(' | ')) + (showTotal ? c.gray(` (${tasks.length})`) : '') } diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index a30f80cc47f7..365262f35a4b 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -26,7 +26,7 @@ interface LocalCallDefinition { end: number name: string type: 'suite' | 'test' - mode: 'run' | 'skip' | 'only' | 'todo' + mode: 'run' | 'skip' | 'only' | 'todo' | 'repeats' task: ParsedSuite | ParsedFile | ParsedTest } From b8690894224cda61445144582e719e19355079f0 Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 26 Jan 2023 15:16:13 -0300 Subject: [PATCH 2/9] Adding test, docs and fixing a bug --- docs/api/index.md | 27 +++++++++++++++++ packages/runner/src/suite.ts | 5 ++++ packages/runner/src/utils/collect.ts | 2 +- test/core/test/repeats.test.ts | 43 ++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/core/test/repeats.test.ts diff --git a/docs/api/index.md b/docs/api/index.md index 9dd5179cfb64..8ddafc22ce52 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -288,6 +288,33 @@ You cannot use this syntax, when using Vitest as [type checker](/guide/testing-t You cannot use this syntax, when using Vitest as [type checker](/guide/testing-types). ::: +### test.repeats + +- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` +- **Alias:** `it.repeats` + + If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. + + By default it will repeat 5 times: + + ```ts + import { expect, test } from 'vitest' + + test.repeats('repeated test', () => { + expect(true).toBe(true) + }) + ``` + + To change the default `repeats` value: + + ```ts + import { expect, test } from 'vitest' + + test.repeats('repeated test', () => { + expect(true).toBe(true) + }, { repeats: 3 }) + ``` + ## bench - **Type:** `(name: string, fn: BenchFunction, options?: BenchOptions) => void` diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index a1547c274d7a..a26698201b8f 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -126,6 +126,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } function initSuite() { + if (typeof suiteOptions === 'number') + suiteOptions = { timeout: suiteOptions } + suite = { id: '', type: 'suite', @@ -133,7 +136,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m mode, shuffle, tasks: [], + repeats: mode === 'repeats' && !suiteOptions?.repeats ? 5 : suiteOptions?.repeats, } + setHooks(suite, createSuiteHooks()) } diff --git a/packages/runner/src/utils/collect.ts b/packages/runner/src/utils/collect.ts index 753797b0a0b2..86a02be3d1ef 100644 --- a/packages/runner/src/utils/collect.ts +++ b/packages/runner/src/utils/collect.ts @@ -39,7 +39,7 @@ export function interpretTaskModes(suite: Suite, namePattern?: string | RegExp, // if all subtasks are skipped, mark as skip if (suite.mode === 'run') { - if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run')) + if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run' && i.mode !== 'repeats')) suite.mode = 'skip' } } diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts new file mode 100644 index 000000000000..78f3d78e9fca --- /dev/null +++ b/test/core/test/repeats.test.ts @@ -0,0 +1,43 @@ +import { afterAll, describe, expect, test } from 'vitest' + +const testNumbers: number[] = [] + +describe('repeat tests', () => { + const result = [1, 1, 1, 1, 1, 2, 2, 2] + // repeats 5 times by default + test.repeats('test 1', () => { + testNumbers.push(1) + }) + + test.repeats('test 2', () => { + testNumbers.push(2) + }, { repeats: 3 }) + + test.repeats.fails('test 3', () => { + testNumbers.push(3) + expect(testNumbers).toStrictEqual(result) + }) + + afterAll(() => { + result.push(3) + expect(testNumbers).toStrictEqual(result) + }) +}) + +const describeNumbers: number[] = [] + +describe.repeats('repeat tests', () => { + test('test 1', () => { + describeNumbers.push(1) + }) +}) + +describe.repeats('repeat tests', () => { + test('test 2', () => { + describeNumbers.push(2) + }) +}, { repeats: 3 }) + +afterAll(() => { + expect(describeNumbers).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2]) +}) From 0f8e50c8f32ed164c67ccb1ee6e9b203f3b1c722 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 27 Feb 2023 10:00:41 -0300 Subject: [PATCH 3/9] Rebase and changing repeated log in terminal --- packages/runner/src/run.ts | 131 ++++++++++-------- packages/runner/src/suite.ts | 2 +- packages/runner/src/types/tasks.ts | 10 +- packages/vitest/src/node/reporters/json.ts | 4 +- .../node/reporters/renderers/listRenderer.ts | 3 + .../src/node/reporters/renderers/utils.ts | 2 - packages/vitest/src/typecheck/collect.ts | 2 +- test/core/test/repeats.test.ts | 6 +- 8 files changed, 94 insertions(+), 66 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 205720cec14d..72d7953d35de 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -107,7 +107,7 @@ const callCleanupHooks = async (cleanups: HookCleanupCallback[]) => { export async function runTest(test: Test, runner: VitestRunner) { await runner.onBeforeRunTest?.(test) - if (test.mode !== 'run') + if (test.mode !== 'run' && test.mode !== 'repeats') return if (test.result?.state === 'fail') { @@ -125,7 +125,7 @@ export async function runTest(test: Test, runner: VitestRunner) { setCurrentTest(test) - const retry = test.retry || 1 + const retry = test.mode === 'repeats' ? test.repeats! : test.retry || 1 for (let retryCount = 0; retryCount < retry; retryCount++) { let beforeEachCleanups: HookCleanupCallback[] = [] try { @@ -147,7 +147,10 @@ export async function runTest(test: Test, runner: VitestRunner) { await runner.onAfterTryTest?.(test, retryCount) - test.result.state = 'pass' + if (test.mode === 'run') + test.result.state = 'pass' + else if (test.mode === 'repeats' && retry === retryCount) + test.result.state = 'pass' } catch (e) { failTask(test.result, e) @@ -164,6 +167,9 @@ export async function runTest(test: Test, runner: VitestRunner) { if (test.result.state === 'pass') break + if (test.mode === 'repeats' && test.result.state === 'fail') + break + // update retry info updateTask(test, runner) } @@ -240,68 +246,83 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { suite.result.state = 'todo' } else { - try { - beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) + let retry = suite.repeats - if (runner.runSuite) { - await runner.runSuite(suite) - } - else { - for (let tasksGroup of partitionSuiteChildren(suite)) { - if (tasksGroup[0].concurrent === true) { - const mutex = limit(runner.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) - } - else { - const { sequence } = runner.config - if (sequence.shuffle || suite.shuffle) { - // run describe block independently from tests - const suites = tasksGroup.filter(group => group.type === 'suite') - const tests = tasksGroup.filter(group => group.type === 'test') - const groups = shuffle([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + for (let retryCount = 0; retryCount < retry!; retryCount++) { + if (suite.mode !== 'repeats') + retry = 1 + try { + beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) + + if (runner.runSuite) { + await runner.runSuite(suite) + } + else { + for (let tasksGroup of partitionSuiteChildren(suite)) { + if (tasksGroup[0].concurrent === true) { + const mutex = limit(runner.config.maxConcurrency) + await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) + } + else { + const { sequence } = runner.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + } + for (const c of tasksGroup) + await runSuiteChild(c, runner) } - for (const c of tasksGroup) - await runSuiteChild(c, runner) } } } - } - catch (e) { - failTask(suite.result, e) - } - - try { - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - await callCleanupHooks(beforeAllCleanups) - } - catch (e) { - failTask(suite.result, e) - } - } + catch (e) { + failTask(suite.result, e) + } - suite.result.duration = now() - start + try { + if (suite.mode !== 'repeats') + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + else if (suite.mode === 'repeats' && retry === retryCount) + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + await callCleanupHooks(beforeAllCleanups) + } + catch (e) { + failTask(suite.result, e) + } - if (suite.mode === 'run') { - if (!hasTests(suite)) { - suite.result.state = 'fail' - if (!suite.result.error) { - const error = processError(new Error(`No test found in suite ${suite.name}`)) - suite.result.error = error - suite.result.errors = [error] + if (suite.mode === 'run') { + if (!hasTests(suite)) { + suite.result.state = 'fail' + if (!suite.result.error) { + const error = processError(new Error(`No test found in suite ${suite.name}`)) + suite.result.error = error + suite.result.errors = [error] + } + } + else if (hasFailed(suite)) { + suite.result.state = 'fail' + } + else { + suite.result.state = 'pass' + } } - } - else if (hasFailed(suite)) { - suite.result.state = 'fail' - } - else { - suite.result.state = 'pass' - } - } - await runner.onAfterRunSuite?.(suite) + updateTask(suite, runner) - updateTask(suite, runner) + suite.result.duration = now() - start + + await runner.onAfterRunSuite?.(suite) + + if (suite.result.state === 'pass') + break + + if (suite.mode === 'repeats' && suite.result.state === 'fail') + break + } + } } async function runSuiteChild(c: Task, runner: VitestRunner) { diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index a26698201b8f..178c15f5829a 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -103,7 +103,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m id: '', name, type: 'custom', - mode: self.only ? 'only' : self.skip ? 'skip' : self.todo ? 'todo' : 'run', + mode: self.only ? 'only' : self.skip ? 'skip' : self.todo ? 'todo' : self.repeats ? 'repeats' : 'run', } tasks.push(task) return task diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 7c306e6d1e7e..ae02285c5a2b 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -16,6 +16,7 @@ export interface TaskBase { result?: TaskResult retry?: number meta?: any + repeats?: number } export interface TaskCustom extends TaskBase { @@ -35,6 +36,7 @@ export interface TaskResult { htmlError?: string hooks?: Partial> retryCount?: number + repeatCount?: number } export type TaskResultPack = [id: string, result: TaskResult | undefined] @@ -161,7 +163,11 @@ export interface TestOptions { * @default 1 */ retry?: number - + /** + * How many times the test will repeat. + * + * @default 5 + */ repeats?: number } @@ -172,7 +178,7 @@ export type TestAPI = ChainableTestAPI & { } type ChainableSuiteAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle', + 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle' | 'repeats', [name: string, factory?: SuiteFactory, options?: number | TestOptions], SuiteCollector, { diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 3b70383fefa3..84bf01aec8dd 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -10,7 +10,7 @@ import { parseErrorStacktrace } from '../../utils/source-map' // the following types are extracted from the Jest repository (and simplified) // the commented-out fields are the missing ones -type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' | 'repeated' +type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' type Milliseconds = number interface Callsite { line: number; column: number } const StatusMap: Record = { @@ -20,7 +20,7 @@ const StatusMap: Record = { run: 'pending', skip: 'skipped', todo: 'todo', - repeats: 'repeated', + repeats: 'pending', } interface FormattedAssertionResult { diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index 5e9ca09c057f..41552c71bc3f 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -107,6 +107,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (task.mode === 'skip' || task.mode === 'todo') suffix += ` ${c.dim(c.gray('[skipped]'))}` + if (task.mode === 'repeats') + suffix += ` ${c.dim(c.gray('[repeated]'))}` + if (task.result?.duration != null) { if (task.result.duration > DURATION_LONG) suffix += c.yellow(` ${Math.round(task.result.duration)}${c.dim('ms')}`) diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 37adb0ea22a4..bb31e14ec173 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -103,13 +103,11 @@ export function getStateString(tasks: Task[], name = 'tests', showTotal = true) const failed = tasks.filter(i => i.result?.state === 'fail') const skipped = tasks.filter(i => i.mode === 'skip') const todo = tasks.filter(i => i.mode === 'todo') - const repeated = tasks.filter(i => i.mode === 'repeats') return [ failed.length ? c.bold(c.red(`${failed.length} failed`)) : null, passed.length ? c.bold(c.green(`${passed.length} passed`)) : null, skipped.length ? c.yellow(`${skipped.length} skipped`) : null, - repeated.length ? c.yellow(`${repeated.length} repeated`) : null, todo.length ? c.gray(`${todo.length} todo`) : null, ].filter(Boolean).join(c.dim(' | ')) + (showTotal ? c.gray(` (${tasks.length})`) : '') } diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index 365262f35a4b..6d3f8222b675 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -83,7 +83,7 @@ export async function collectTests(ctx: Vitest, filepath: string): Promise { +describe('testing it/test', () => { const result = [1, 1, 1, 1, 1, 2, 2, 2] // repeats 5 times by default test.repeats('test 1', () => { @@ -26,13 +26,13 @@ describe('repeat tests', () => { const describeNumbers: number[] = [] -describe.repeats('repeat tests', () => { +describe.repeats('testing describe 1', () => { test('test 1', () => { describeNumbers.push(1) }) }) -describe.repeats('repeat tests', () => { +describe.repeats('testing describe 2', () => { test('test 2', () => { describeNumbers.push(2) }) From 7859a907e64c6ff12c1a33ceac9a177f5e8a30b0 Mon Sep 17 00:00:00 2001 From: Samuel Date: Tue, 7 Mar 2023 09:53:12 -0300 Subject: [PATCH 4/9] Fixing repeat suite logic --- packages/runner/src/run.ts | 6 ++---- .../vitest/src/node/reporters/renderers/listRenderer.ts | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 72d7953d35de..d7491a4b2f0d 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -246,11 +246,9 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { suite.result.state = 'todo' } else { - let retry = suite.repeats + const retry = suite.mode === 'repeats' ? suite.repeats! : suite.retry || 1 - for (let retryCount = 0; retryCount < retry!; retryCount++) { - if (suite.mode !== 'repeats') - retry = 1 + for (let retryCount = 0; retryCount < retry; retryCount++) { try { beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index 41552c71bc3f..13309bb3d40c 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -107,7 +107,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (task.mode === 'skip' || task.mode === 'todo') suffix += ` ${c.dim(c.gray('[skipped]'))}` - if (task.mode === 'repeats') + if (task.type === 'suite' && task.tasks.find(t => t.mode === 'repeats')) suffix += ` ${c.dim(c.gray('[repeated]'))}` if (task.result?.duration != null) { From bdec0b194780acfa0edfea75db1a58fede42f930 Mon Sep 17 00:00:00 2001 From: Samuel Date: Wed, 8 Mar 2023 16:12:03 -0300 Subject: [PATCH 5/9] Changing repeat and retry logic --- docs/api/index.md | 14 +-- packages/runner/src/run.ts | 201 +++++++++++++++++---------------- packages/runner/src/suite.ts | 5 +- test/core/test/repeats.test.ts | 27 +++-- 4 files changed, 125 insertions(+), 122 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 8ddafc22ce52..d066176bda8a 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -293,19 +293,7 @@ You cannot use this syntax, when using Vitest as [type checker](/guide/testing-t - **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` - **Alias:** `it.repeats` - If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. - - By default it will repeat 5 times: - - ```ts - import { expect, test } from 'vitest' - - test.repeats('repeated test', () => { - expect(true).toBe(true) - }) - ``` - - To change the default `repeats` value: + If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. Without the `repeats` options it will only run once. ```ts import { expect, test } from 'vitest' diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index d7491a4b2f0d..f27af6732699 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -125,53 +125,58 @@ export async function runTest(test: Test, runner: VitestRunner) { setCurrentTest(test) - const retry = test.mode === 'repeats' ? test.repeats! : test.retry || 1 - for (let retryCount = 0; retryCount < retry; retryCount++) { - let beforeEachCleanups: HookCleanupCallback[] = [] - try { - await runner.onBeforeTryTest?.(test, retryCount) + const repeats = test.mode === 'repeats' ? test.repeats || 1 : 1 - beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) + for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { + const retry = test.retry || 1 - test.result.retryCount = retryCount + for (let retryCount = 0; retryCount < retry; retryCount++) { + let beforeEachCleanups: HookCleanupCallback[] = [] + try { + await runner.onBeforeTryTest?.(test, retryCount) - if (runner.runTest) { - await runner.runTest(test) - } - else { - const fn = getFn(test) - if (!fn) - throw new Error('Test function is not found. Did you add it using `setFn`?') - await fn() - } + beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) - await runner.onAfterTryTest?.(test, retryCount) + test.result.retryCount = retryCount - if (test.mode === 'run') - test.result.state = 'pass' - else if (test.mode === 'repeats' && retry === retryCount) - test.result.state = 'pass' - } - catch (e) { - failTask(test.result, e) - } + if (runner.runTest) { + await runner.runTest(test) + } + else { + const fn = getFn(test) + if (!fn) + throw new Error('Test function is not found. Did you add it using `setFn`?') + await fn() + } - try { - await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite]) - await callCleanupHooks(beforeEachCleanups) - } - catch (e) { - failTask(test.result, e) - } + await runner.onAfterTryTest?.(test, retryCount) + + if (test.mode === 'run') + test.result.state = 'pass' + else if (test.mode === 'repeats' && retry === retryCount) + test.result.state = 'pass' + } + catch (e) { + failTask(test.result, e) + } - if (test.result.state === 'pass') - break + try { + await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite]) + await callCleanupHooks(beforeEachCleanups) + } + catch (e) { + failTask(test.result, e) + } - if (test.mode === 'repeats' && test.result.state === 'fail') - break + if (test.result.state === 'pass') + break - // update retry info - updateTask(test, runner) + if (test.mode === 'repeats' && retry === retryCount && test.result.state === 'fail') + break + + // update retry info + updateTask(test, runner) + } } if (test.result.state === 'fail') @@ -246,79 +251,83 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { suite.result.state = 'todo' } else { - const retry = suite.mode === 'repeats' ? suite.repeats! : suite.retry || 1 + const repeats = suite.mode === 'repeats' ? suite.repeats || 1 : 1 - for (let retryCount = 0; retryCount < retry; retryCount++) { - try { - beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) + for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { + const retry = suite.retry || 1 - if (runner.runSuite) { - await runner.runSuite(suite) - } - else { - for (let tasksGroup of partitionSuiteChildren(suite)) { - if (tasksGroup[0].concurrent === true) { - const mutex = limit(runner.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) - } - else { - const { sequence } = runner.config - if (sequence.shuffle || suite.shuffle) { - // run describe block independently from tests - const suites = tasksGroup.filter(group => group.type === 'suite') - const tests = tasksGroup.filter(group => group.type === 'test') - const groups = shuffle([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + for (let retryCount = 0; retryCount < retry; retryCount++) { + try { + beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) + + if (runner.runSuite) { + await runner.runSuite(suite) + } + else { + for (let tasksGroup of partitionSuiteChildren(suite)) { + if (tasksGroup[0].concurrent === true) { + const mutex = limit(runner.config.maxConcurrency) + await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) + } + else { + const { sequence } = runner.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + } + for (const c of tasksGroup) + await runSuiteChild(c, runner) } - for (const c of tasksGroup) - await runSuiteChild(c, runner) } } } - } - catch (e) { - failTask(suite.result, e) - } - - try { - if (suite.mode !== 'repeats') - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - else if (suite.mode === 'repeats' && retry === retryCount) - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - await callCleanupHooks(beforeAllCleanups) - } - catch (e) { - failTask(suite.result, e) - } + catch (e) { + failTask(suite.result, e) + } - if (suite.mode === 'run') { - if (!hasTests(suite)) { - suite.result.state = 'fail' - if (!suite.result.error) { - const error = processError(new Error(`No test found in suite ${suite.name}`)) - suite.result.error = error - suite.result.errors = [error] - } + try { + if (suite.mode !== 'repeats') + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + else if (suite.mode === 'repeats' && repeatCount - 1 === repeats && retry === retryCount) + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + await callCleanupHooks(beforeAllCleanups) } - else if (hasFailed(suite)) { - suite.result.state = 'fail' + catch (e) { + failTask(suite.result, e) } - else { - suite.result.state = 'pass' + + if (suite.mode === 'run') { + if (!hasTests(suite)) { + suite.result.state = 'fail' + if (!suite.result.error) { + const error = processError(new Error(`No test found in suite ${suite.name}`)) + suite.result.error = error + suite.result.errors = [error] + } + } + else if (hasFailed(suite)) { + suite.result.state = 'fail' + } + else { + suite.result.state = 'pass' + } } - } - updateTask(suite, runner) + updateTask(suite, runner) - suite.result.duration = now() - start + suite.result.duration = now() - start - await runner.onAfterRunSuite?.(suite) + await runner.onAfterRunSuite?.(suite) - if (suite.result.state === 'pass') - break + if (suite.result.state === 'pass') + break - if (suite.mode === 'repeats' && suite.result.state === 'fail') - break + if (suite.mode === 'repeats' && suite.result.state === 'fail') + break + } } } } diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index 178c15f5829a..aa13b7a7f290 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -73,8 +73,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m suite: undefined!, fails: this.fails, retry: options?.retry, - // 5 repetitions by default - repeats: mode === 'repeats' && !options?.repeats ? 5 : options?.repeats, + repeats: options?.repeats, } as Omit as Test if (this.concurrent || concurrent) @@ -136,7 +135,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m mode, shuffle, tasks: [], - repeats: mode === 'repeats' && !suiteOptions?.repeats ? 5 : suiteOptions?.repeats, + repeats: suiteOptions?.repeats, } setHooks(suite, createSuiteHooks()) diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts index 7e27d47d6ad2..3f55d76fff71 100644 --- a/test/core/test/repeats.test.ts +++ b/test/core/test/repeats.test.ts @@ -4,10 +4,10 @@ const testNumbers: number[] = [] describe('testing it/test', () => { const result = [1, 1, 1, 1, 1, 2, 2, 2] - // repeats 5 times by default + test.repeats('test 1', () => { testNumbers.push(1) - }) + }, { repeats: 5 }) test.repeats('test 2', () => { testNumbers.push(2) @@ -26,18 +26,25 @@ describe('testing it/test', () => { const describeNumbers: number[] = [] -describe.repeats('testing describe 1', () => { +describe.repeats('testing describe', () => { test('test 1', () => { describeNumbers.push(1) }) -}) - -describe.repeats('testing describe 2', () => { - test('test 2', () => { - describeNumbers.push(2) - }) }, { repeats: 3 }) afterAll(() => { - expect(describeNumbers).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2]) + expect(describeNumbers).toStrictEqual([1, 1, 1]) +}) + +const retryNumbers: number[] = [] + +describe('testing repeats with retry', () => { + const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + test.repeats('test 1', () => { + retryNumbers.push(1) + }, { repeats: 5, retry: 2 }) + + afterAll(() => { + expect(retryNumbers).toStrictEqual(result) + }) }) From 1fa87954557fff040b05c6368cc330f3a84a537b Mon Sep 17 00:00:00 2001 From: Samuel Date: Thu, 9 Mar 2023 11:02:07 -0300 Subject: [PATCH 6/9] Removing repeats mode --- packages/runner/src/run.ts | 129 ++++++++---------- packages/runner/src/suite.ts | 5 +- packages/runner/src/types/tasks.ts | 2 +- packages/runner/src/utils/collect.ts | 2 +- packages/vitest/src/node/reporters/json.ts | 1 - .../node/reporters/renderers/listRenderer.ts | 4 +- packages/vitest/src/typecheck/collect.ts | 2 +- test/core/test/repeats.test.ts | 2 +- 8 files changed, 66 insertions(+), 81 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index f27af6732699..b15b1cba948c 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -107,7 +107,7 @@ const callCleanupHooks = async (cleanups: HookCleanupCallback[]) => { export async function runTest(test: Test, runner: VitestRunner) { await runner.onBeforeRunTest?.(test) - if (test.mode !== 'run' && test.mode !== 'repeats') + if (test.mode !== 'run' && !test.repeats) return if (test.result?.state === 'fail') { @@ -125,7 +125,7 @@ export async function runTest(test: Test, runner: VitestRunner) { setCurrentTest(test) - const repeats = test.mode === 'repeats' ? test.repeats || 1 : 1 + const repeats = test.repeats ? test.repeats || 1 : 1 for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { const retry = test.retry || 1 @@ -138,6 +138,7 @@ export async function runTest(test: Test, runner: VitestRunner) { beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) test.result.retryCount = retryCount + test.result.repeatCount = repeatCount if (runner.runTest) { await runner.runTest(test) @@ -151,9 +152,9 @@ export async function runTest(test: Test, runner: VitestRunner) { await runner.onAfterTryTest?.(test, retryCount) - if (test.mode === 'run') + if (!test.repeats) test.result.state = 'pass' - else if (test.mode === 'repeats' && retry === retryCount) + else if (test.repeats && retry === retryCount) test.result.state = 'pass' } catch (e) { @@ -170,10 +171,6 @@ export async function runTest(test: Test, runner: VitestRunner) { if (test.result.state === 'pass') break - - if (test.mode === 'repeats' && retry === retryCount && test.result.state === 'fail') - break - // update retry info updateTask(test, runner) } @@ -251,83 +248,73 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { suite.result.state = 'todo' } else { - const repeats = suite.mode === 'repeats' ? suite.repeats || 1 : 1 - - for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { - const retry = suite.retry || 1 + const retry = suite.retry || 1 - for (let retryCount = 0; retryCount < retry; retryCount++) { - try { - beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) + for (let retryCount = 0; retryCount < retry; retryCount++) { + try { + beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) - if (runner.runSuite) { - await runner.runSuite(suite) - } - else { - for (let tasksGroup of partitionSuiteChildren(suite)) { - if (tasksGroup[0].concurrent === true) { - const mutex = limit(runner.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) - } - else { - const { sequence } = runner.config - if (sequence.shuffle || suite.shuffle) { - // run describe block independently from tests - const suites = tasksGroup.filter(group => group.type === 'suite') - const tests = tasksGroup.filter(group => group.type === 'test') - const groups = shuffle([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) - } - for (const c of tasksGroup) - await runSuiteChild(c, runner) + if (runner.runSuite) { + await runner.runSuite(suite) + } + else { + for (let tasksGroup of partitionSuiteChildren(suite)) { + if (tasksGroup[0].concurrent === true) { + const mutex = limit(runner.config.maxConcurrency) + await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) + } + else { + const { sequence } = runner.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) } + for (const c of tasksGroup) + await runSuiteChild(c, runner) } } } - catch (e) { - failTask(suite.result, e) - } + } + catch (e) { + failTask(suite.result, e) + } - try { - if (suite.mode !== 'repeats') - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - else if (suite.mode === 'repeats' && repeatCount - 1 === repeats && retry === retryCount) - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - await callCleanupHooks(beforeAllCleanups) - } - catch (e) { - failTask(suite.result, e) - } + try { + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + await callCleanupHooks(beforeAllCleanups) + } + catch (e) { + failTask(suite.result, e) + } - if (suite.mode === 'run') { - if (!hasTests(suite)) { - suite.result.state = 'fail' - if (!suite.result.error) { - const error = processError(new Error(`No test found in suite ${suite.name}`)) - suite.result.error = error - suite.result.errors = [error] - } - } - else if (hasFailed(suite)) { - suite.result.state = 'fail' - } - else { - suite.result.state = 'pass' + if (suite.mode === 'run') { + if (!hasTests(suite)) { + suite.result.state = 'fail' + if (!suite.result.error) { + const error = processError(new Error(`No test found in suite ${suite.name}`)) + suite.result.error = error + suite.result.errors = [error] } } + else if (hasFailed(suite)) { + suite.result.state = 'fail' + } + else { + suite.result.state = 'pass' + } + } - updateTask(suite, runner) - - suite.result.duration = now() - start + updateTask(suite, runner) - await runner.onAfterRunSuite?.(suite) + suite.result.duration = now() - start - if (suite.result.state === 'pass') - break + await runner.onAfterRunSuite?.(suite) - if (suite.mode === 'repeats' && suite.result.state === 'fail') - break - } + if (suite.result.state === 'pass') + break } } } diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index aa13b7a7f290..cbea06938f10 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -102,7 +102,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m id: '', name, type: 'custom', - mode: self.only ? 'only' : self.skip ? 'skip' : self.todo ? 'todo' : self.repeats ? 'repeats' : 'run', + mode: self.only ? 'only' : self.skip ? 'skip' : self.todo ? 'todo' : 'run', } tasks.push(task) return task @@ -135,7 +135,6 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m mode, shuffle, tasks: [], - repeats: suiteOptions?.repeats, } setHooks(suite, createSuiteHooks()) @@ -175,7 +174,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { function suiteFn(this: Record, name: string, factory?: SuiteFactory, options?: number | TestOptions) { - const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' + const mode: RunMode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle, options) } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index ae02285c5a2b..40fa09931d55 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -2,7 +2,7 @@ import type { Awaitable } from '@vitest/utils' import type { ChainableFunction } from '../utils/chain' import type { ErrorWithDiff } from '../utils/error' -export type RunMode = 'run' | 'skip' | 'only' | 'todo' | 'repeats' +export type RunMode = 'run' | 'skip' | 'only' | 'todo' export type TaskState = RunMode | 'pass' | 'fail' export interface TaskBase { diff --git a/packages/runner/src/utils/collect.ts b/packages/runner/src/utils/collect.ts index 86a02be3d1ef..bc4b128d9e77 100644 --- a/packages/runner/src/utils/collect.ts +++ b/packages/runner/src/utils/collect.ts @@ -39,7 +39,7 @@ export function interpretTaskModes(suite: Suite, namePattern?: string | RegExp, // if all subtasks are skipped, mark as skip if (suite.mode === 'run') { - if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run' && i.mode !== 'repeats')) + if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run' && !i.repeats)) suite.mode = 'skip' } } diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 84bf01aec8dd..533255288736 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -20,7 +20,6 @@ const StatusMap: Record = { run: 'pending', skip: 'skipped', todo: 'todo', - repeats: 'pending', } interface FormattedAssertionResult { diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index 13309bb3d40c..a9fc2475f6c1 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -107,8 +107,8 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (task.mode === 'skip' || task.mode === 'todo') suffix += ` ${c.dim(c.gray('[skipped]'))}` - if (task.type === 'suite' && task.tasks.find(t => t.mode === 'repeats')) - suffix += ` ${c.dim(c.gray('[repeated]'))}` + if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1) + suffix += c.yellow(` (repeat x${task.result.repeatCount})`) if (task.result?.duration != null) { if (task.result.duration > DURATION_LONG) diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index 6d3f8222b675..1fb7e1bd4fa4 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -26,7 +26,7 @@ interface LocalCallDefinition { end: number name: string type: 'suite' | 'test' - mode: 'run' | 'skip' | 'only' | 'todo' | 'repeats' + mode: 'run' | 'skip' | 'only' | 'todo' task: ParsedSuite | ParsedFile | ParsedTest } diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts index 3f55d76fff71..c2039e327233 100644 --- a/test/core/test/repeats.test.ts +++ b/test/core/test/repeats.test.ts @@ -16,7 +16,7 @@ describe('testing it/test', () => { test.repeats.fails('test 3', () => { testNumbers.push(3) expect(testNumbers).toStrictEqual(result) - }) + }, { repeats: 1 }) afterAll(() => { result.push(3) From cf8febfe80c792ab2d1d28c96049b7d632bfa065 Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 10 Mar 2023 12:44:07 -0300 Subject: [PATCH 7/9] Updaing snapshot --- packages/runner/src/run.ts | 2 +- packages/runner/src/suite.ts | 2 +- packages/runner/src/utils/collect.ts | 2 +- test/reporters/tests/__snapshots__/html.test.ts.snap | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index b15b1cba948c..af1f81c7f425 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -107,7 +107,7 @@ const callCleanupHooks = async (cleanups: HookCleanupCallback[]) => { export async function runTest(test: Test, runner: VitestRunner) { await runner.onBeforeRunTest?.(test) - if (test.mode !== 'run' && !test.repeats) + if (test.mode !== 'run') return if (test.result?.state === 'fail') { diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index cbea06938f10..612e15396c58 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -60,7 +60,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m initSuite() const test = createTest(function (name: string, fn = noop, options = suiteOptions) { - const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : this.repeats ? 'repeats' : 'run' + const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' if (typeof options === 'number') options = { timeout: options } diff --git a/packages/runner/src/utils/collect.ts b/packages/runner/src/utils/collect.ts index bc4b128d9e77..753797b0a0b2 100644 --- a/packages/runner/src/utils/collect.ts +++ b/packages/runner/src/utils/collect.ts @@ -39,7 +39,7 @@ export function interpretTaskModes(suite: Suite, namePattern?: string | RegExp, // if all subtasks are skipped, mark as skip if (suite.mode === 'run') { - if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run' && !i.repeats)) + if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run')) suite.mode = 'skip' } } diff --git a/test/reporters/tests/__snapshots__/html.test.ts.snap b/test/reporters/tests/__snapshots__/html.test.ts.snap index 49db8c36242b..22c59eb0a97c 100644 --- a/test/reporters/tests/__snapshots__/html.test.ts.snap +++ b/test/reporters/tests/__snapshots__/html.test.ts.snap @@ -72,6 +72,7 @@ exports[`html reporter > resolves to "failing" status for test file "json-fail" "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "fail", @@ -141,6 +142,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "pass", @@ -187,6 +189,7 @@ exports[`html reporter > resolves to "passing" status for test file "all-passing "afterEach": "pass", "beforeEach": "pass", }, + "repeatCount": 0, "retryCount": 0, "startTime": 0, "state": "pass", From 312e4f9116ce2e917da49693f23e0f5c0b2c62d8 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Fri, 14 Apr 2023 18:38:18 +0200 Subject: [PATCH 8/9] chore: remove "repeats" keyword --- docs/api/index.md | 15 --- packages/runner/src/run.ts | 131 +++++++++++------------ packages/runner/src/suite.ts | 14 ++- packages/runner/src/types/runner.ts | 4 +- packages/runner/src/types/tasks.ts | 4 +- packages/vitest/src/typecheck/collect.ts | 2 +- test/core/test/repeats.test.ts | 10 +- 7 files changed, 82 insertions(+), 98 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index a0fc9a623d5e..38b74aa68272 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -295,21 +295,6 @@ Vitest processes `$values` with chai `format` method. If the value is too trunca You cannot use this syntax, when using Vitest as [type checker](/guide/testing-types). ::: -### test.repeats - -- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` -- **Alias:** `it.repeats` - - If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. Without the `repeats` options it will only run once. - - ```ts - import { expect, test } from 'vitest' - - test.repeats('repeated test', () => { - expect(true).toBe(true) - }, { repeats: 3 }) - ``` - ## bench - **Type:** `(name: string, fn: BenchFunction, options?: BenchOptions) => void` diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index f686088021d1..9eaaa97a4e1d 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -125,7 +125,7 @@ export async function runTest(test: Test, runner: VitestRunner) { setCurrentTest(test) - const repeats = test.repeats ? test.repeats || 1 : 1 + const repeats = typeof test.repeats === 'number' ? test.repeats : 1 for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { const retry = test.retry || 1 @@ -133,22 +133,12 @@ export async function runTest(test: Test, runner: VitestRunner) { for (let retryCount = 0; retryCount < retry; retryCount++) { let beforeEachCleanups: HookCleanupCallback[] = [] try { - await runner.onBeforeTryTest?.(test, retryCount) + await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount }) - beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) - - // some async expect will be added to this array, in case user forget to await theme - if (test.promises) { - const result = await Promise.allSettled(test.promises) - const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean) - if (errors.length) - throw errors - } - test.result.retryCount = retryCount test.result.repeatCount = repeatCount - await runner.onAfterTryTest?.(test, retryCount) + beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) if (runner.runTest) { await runner.runTest(test) @@ -160,7 +150,15 @@ export async function runTest(test: Test, runner: VitestRunner) { await fn() } - await runner.onAfterTryTest?.(test, retryCount) + // some async expect will be added to this array, in case user forget to await theme + if (test.promises) { + const result = await Promise.allSettled(test.promises) + const errors = result.map(r => r.status === 'rejected' ? r.reason : undefined).filter(Boolean) + if (errors.length) + throw errors + } + + await runner.onAfterTryTest?.(test, { retry: retryCount, repeats: repeatCount }) if (!test.repeats) test.result.state = 'pass' @@ -263,74 +261,67 @@ export async function runSuite(suite: Suite, runner: VitestRunner) { suite.result.state = 'todo' } else { - const retry = suite.retry || 1 + try { + beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) - for (let retryCount = 0; retryCount < retry; retryCount++) { - try { - beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite]) - - if (runner.runSuite) { - await runner.runSuite(suite) - } - else { - for (let tasksGroup of partitionSuiteChildren(suite)) { - if (tasksGroup[0].concurrent === true) { - const mutex = limit(runner.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) - } - else { - const { sequence } = runner.config - if (sequence.shuffle || suite.shuffle) { - // run describe block independently from tests - const suites = tasksGroup.filter(group => group.type === 'suite') - const tests = tasksGroup.filter(group => group.type === 'test') - const groups = shuffle([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) - } - for (const c of tasksGroup) - await runSuiteChild(c, runner) + if (runner.runSuite) { + await runner.runSuite(suite) + } + else { + for (let tasksGroup of partitionSuiteChildren(suite)) { + if (tasksGroup[0].concurrent === true) { + const mutex = limit(runner.config.maxConcurrency) + await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner)))) + } + else { + const { sequence } = runner.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) } + for (const c of tasksGroup) + await runSuiteChild(c, runner) } } } - catch (e) { - failTask(suite.result, e) - } + } + catch (e) { + failTask(suite.result, e) + } - try { - await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) - await callCleanupHooks(beforeAllCleanups) - } - catch (e) { - failTask(suite.result, e) - } + try { + await callSuiteHook(suite, suite, 'afterAll', runner, [suite]) + await callCleanupHooks(beforeAllCleanups) + } + catch (e) { + failTask(suite.result, e) + } - if (suite.mode === 'run') { - if (!hasTests(suite)) { - suite.result.state = 'fail' - if (!suite.result.error) { - const error = processError(new Error(`No test found in suite ${suite.name}`)) - suite.result.error = error - suite.result.errors = [error] - } - } - else if (hasFailed(suite)) { - suite.result.state = 'fail' - } - else { - suite.result.state = 'pass' + if (suite.mode === 'run') { + if (!hasTests(suite)) { + suite.result.state = 'fail' + if (!suite.result.error) { + const error = processError(new Error(`No test found in suite ${suite.name}`)) + suite.result.error = error + suite.result.errors = [error] } } + else if (hasFailed(suite)) { + suite.result.state = 'fail' + } + else { + suite.result.state = 'pass' + } + } - updateTask(suite, runner) - - suite.result.duration = now() - start + updateTask(suite, runner) - await runner.onAfterRunSuite?.(suite) + suite.result.duration = now() - start - if (suite.result.state === 'pass') - break - } + await runner.onAfterRunSuite?.(suite) } } diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index ce06890bf214..16fd20232f02 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -65,6 +65,14 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m if (typeof options === 'number') options = { timeout: options } + // inherit repeats and retry from suite + if (typeof suiteOptions === 'object') { + options = { + ...suiteOptions, + ...options, + } + } + const test: Test = { id: '', type: 'test', @@ -199,14 +207,14 @@ function createSuite() { suiteFn.runIf = (condition: any) => (condition ? suite : suite.skip) as SuiteAPI return createChainable( - ['concurrent', 'shuffle', 'skip', 'only', 'todo', 'repeats'], + ['concurrent', 'shuffle', 'skip', 'only', 'todo'], suiteFn, ) as unknown as SuiteAPI } function createTest(fn: ( ( - this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails' | 'repeats', boolean | undefined>, + this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>, title: string, fn?: TestFunction, options?: number | TestOptions @@ -236,7 +244,7 @@ function createTest(fn: ( testFn.runIf = (condition: any) => (condition ? test : test.skip) as TestAPI return createChainable( - ['concurrent', 'skip', 'only', 'todo', 'fails', 'repeats'], + ['concurrent', 'skip', 'only', 'todo', 'fails'], testFn, ) as TestAPI } diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 382b7eee04c1..be651843674f 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -44,7 +44,7 @@ export interface VitestRunner { /** * Called before actually running the test function. Already has "result" with "state" and "startTime". */ - onBeforeTryTest?(test: Test, retryCount: number): unknown + onBeforeTryTest?(test: Test, options: { retry: number; repeats: number }): unknown /** * Called after result and state are set. */ @@ -52,7 +52,7 @@ export interface VitestRunner { /** * Called right after running the test function. Doesn't have new state yet. Will not be called, if the test function throws. */ - onAfterTryTest?(test: Test, retryCount: number): unknown + onAfterTryTest?(test: Test, options: { retry: number; repeats: number }): unknown /** * Called before running a single suite. Doesn't have "result" yet. diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index d1f56f1fc66f..5c548d87240d 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -146,7 +146,7 @@ interface TestEachFunction { } type ChainableTestAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'fails' | 'repeats', + 'concurrent' | 'only' | 'skip' | 'todo' | 'fails', [name: string, fn?: TestFunction, options?: number | TestOptions], void, { @@ -182,7 +182,7 @@ export type TestAPI = ChainableTestAPI & { } type ChainableSuiteAPI = ChainableFunction< - 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle' | 'repeats', + 'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle', [name: string, factory?: SuiteFactory, options?: number | TestOptions], SuiteCollector, { diff --git a/packages/vitest/src/typecheck/collect.ts b/packages/vitest/src/typecheck/collect.ts index ddc9bb510b16..97e079b44a0a 100644 --- a/packages/vitest/src/typecheck/collect.ts +++ b/packages/vitest/src/typecheck/collect.ts @@ -85,7 +85,7 @@ export async function collectTests(ctx: WorkspaceProject, filepath: string): Pro const { arguments: [{ value: message }] } = node as any const property = callee?.property?.name let mode = (!property || property === name) ? 'run' : property - if (!['run', 'skip', 'todo', 'only', 'skipIf', 'runIf', 'repeats'].includes(mode)) + if (!['run', 'skip', 'todo', 'only', 'skipIf', 'runIf'].includes(mode)) throw new Error(`${name}.${mode} syntax is not supported when testing types`) // cannot statically analyze, so we always skip it if (mode === 'skipIf' || mode === 'runIf') diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts index c2039e327233..7542ceada924 100644 --- a/test/core/test/repeats.test.ts +++ b/test/core/test/repeats.test.ts @@ -5,15 +5,15 @@ const testNumbers: number[] = [] describe('testing it/test', () => { const result = [1, 1, 1, 1, 1, 2, 2, 2] - test.repeats('test 1', () => { + test('test 1', () => { testNumbers.push(1) }, { repeats: 5 }) - test.repeats('test 2', () => { + test('test 2', () => { testNumbers.push(2) }, { repeats: 3 }) - test.repeats.fails('test 3', () => { + test.fails('test 3', () => { testNumbers.push(3) expect(testNumbers).toStrictEqual(result) }, { repeats: 1 }) @@ -26,7 +26,7 @@ describe('testing it/test', () => { const describeNumbers: number[] = [] -describe.repeats('testing describe', () => { +describe('testing describe', () => { test('test 1', () => { describeNumbers.push(1) }) @@ -40,7 +40,7 @@ const retryNumbers: number[] = [] describe('testing repeats with retry', () => { const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - test.repeats('test 1', () => { + test('test 1', () => { retryNumbers.push(1) }, { repeats: 5, retry: 2 }) From 12e0f85209e6e8afb8f4a8e4ab031cda2a74407e Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Fri, 14 Apr 2023 18:46:41 +0200 Subject: [PATCH 9/9] docs: add repeats to docs --- docs/api/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/api/index.md b/docs/api/index.md index 38b74aa68272..5da908cc8b25 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11,8 +11,20 @@ type Awaitable = T | PromiseLike type TestFunction = () => Awaitable interface TestOptions { + /** + * Will fail the test if it takes too long to execute + */ timeout?: number + /** + * Will retry the test specific number of times if it fails + */ retry?: number + /** + * Will repeat the same test several times even if it fails each time + * If you have "retry" option and it fails, it will use every retry in each cycle + * Useful for debugging random failings + */ + repeats?: number } ```