diff --git a/docs/api/index.md b/docs/api/index.md index 143807b500fd..d08ec1b7bbd6 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -17,12 +17,16 @@ interface TestOptions { timeout?: number /** * Will retry the test specific number of times if it fails + * + * @default 0 */ retry?: number /** * Will repeat the same test several times even if it fails each time * If you have "retry" option and it fails, it will use every retry in each cycle * Useful for debugging random failings + * + * @default 0 */ repeats?: number } diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 1788675632e6..a97b3e64df7b 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -127,22 +127,20 @@ export async function runTest(test: Test, runner: VitestRunner) { test.result = { state: 'run', startTime: start, + retryCount: 0, } updateTask(test, runner) setCurrentTest(test) - const repeats = typeof test.repeats === 'number' ? test.repeats : 1 - - for (let repeatCount = 0; repeatCount < repeats; repeatCount++) { - const retry = test.retry || 1 - - for (let retryCount = 0; retryCount < retry; retryCount++) { + const repeats = test.repeats ?? 0 + for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) { + const retry = test.retry ?? 0 + for (let retryCount = 0; retryCount <= retry; retryCount++) { let beforeEachCleanups: HookCleanupCallback[] = [] try { await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount }) - test.result.retryCount = retryCount test.result.repeatCount = repeatCount beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite]) @@ -188,9 +186,10 @@ export async function runTest(test: Test, runner: VitestRunner) { if (test.result.state === 'pass') break - if (retryCount < retry - 1) { + if (retryCount < retry) { // reset state when retry test test.result.state = 'run' + test.result.retryCount = (test.result.retryCount ?? 0) + 1 } // update retry info diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 04c6cf03ce37..b1e41e46d08b 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -166,15 +166,14 @@ export interface TestOptions { * Times to retry the test if fails. Useful for making flaky tests more stable. * When retries is up, the last test error will be thrown. * - * @default 1 + * @default 0 */ retry?: number /** * How many times the test will run. * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default. * - * @default 1 - * + * @default 0 */ repeats?: number } diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index d27c25cdd438..254256da5614 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -100,7 +100,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (level === 0 && task.type === 'suite' && task.projectName) prefix += formatProjectName(task.projectName) - if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1) + if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 0) suffix += c.yellow(` (retry x${task.result.retryCount})`) if (task.type === 'suite' && !task.meta?.typecheck) { @@ -111,7 +111,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = if (task.mode === 'skip' || task.mode === 'todo') suffix += ` ${c.dim(c.gray('[skipped]'))}` - if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1) + if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 0) suffix += c.yellow(` (repeat x${task.result.repeatCount})`) if (task.result?.duration != null) { diff --git a/test/core/test/propagate-options-nested-suite.test.ts b/test/core/test/propagate-options-nested-suite.test.ts index 4c10b183e508..d7b11fd44403 100644 --- a/test/core/test/propagate-options-nested-suite.test.ts +++ b/test/core/test/propagate-options-nested-suite.test.ts @@ -19,5 +19,5 @@ describe( }) }) }, - { retry: 10 }, + { retry: 9 }, ) diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts index 711319440bab..ad12bfc0914b 100644 --- a/test/core/test/repeats.test.ts +++ b/test/core/test/repeats.test.ts @@ -1,3 +1,4 @@ +import { getCurrentTest } from '@vitest/runner' import { afterAll, describe, expect, test } from 'vitest' const testNumbers: number[] = [] @@ -7,16 +8,16 @@ describe('testing it/test', () => { test('test 1', () => { testNumbers.push(1) - }, { repeats: 5 }) + }, { repeats: 4 }) test('test 2', () => { testNumbers.push(2) - }, { repeats: 3 }) + }, { repeats: 2 }) test.fails('test 3', () => { testNumbers.push(3) expect(testNumbers).toStrictEqual(result) - }, { repeats: 1 }) + }, { repeats: 0 }) afterAll(() => { result.push(3) @@ -30,7 +31,7 @@ describe('testing describe', () => { test('test 1', () => { describeNumbers.push(1) }) -}, { repeats: 3 }) +}, { repeats: 2 }) afterAll(() => { expect(describeNumbers).toStrictEqual([1, 1, 1]) @@ -39,14 +40,21 @@ afterAll(() => { const retryNumbers: number[] = [] describe('testing repeats with retry', () => { - const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - test('test 1', () => { - retryNumbers.push(1) - }, { repeats: 5, retry: 2 }) - - afterAll(() => { - expect(retryNumbers).toStrictEqual(result) + describe('normal test', () => { + const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + test.fails('test 1', () => { + retryNumbers.push(1) + expect(1).toBe(2) + }, { repeats: 4, retry: 1 }) + + afterAll(() => { + expect(retryNumbers).toStrictEqual(result) + }) }) + + test('should not reset retry count', () => { + expect(getCurrentTest()!.result?.retryCount).toBe(3) + }, { repeats: 2, retry: 1 }) }) const nestedDescribeNumbers: number[] = [] @@ -71,10 +79,10 @@ describe('testing nested describe', () => { nestedDescribeNumbers.push(4) }) }, 100) - }, { repeats: 3 }) + }, { repeats: 2 }) }) afterAll(() => { expect(nestedDescribeNumbers).toStrictEqual([1, 1, 2, 2, 3, 3, 3, 4, 4, 4]) }) -}, { repeats: 2 }) +}, { repeats: 1 }) diff --git a/test/core/test/retry-only.test.ts b/test/core/test/retry-only.test.ts index a906c8f6bb27..8a8a2b0df54c 100644 --- a/test/core/test/retry-only.test.ts +++ b/test/core/test/retry-only.test.ts @@ -11,5 +11,5 @@ describe.only('description.only retry', () => { it('test should not inherit options from the description block if exists', () => { count5 += 1 expect(count5).toBe(5) - }, { retry: 5 }) -}, { retry: 2 }) + }, { retry: 4 }) +}, { retry: 1 }) diff --git a/test/core/test/retry.test.ts b/test/core/test/retry.test.ts index a60926b5e210..9a40879c4d24 100644 --- a/test/core/test/retry.test.ts +++ b/test/core/test/retry.test.ts @@ -4,13 +4,13 @@ let count1 = 0 it('retry test', () => { count1 += 1 expect(count1).toBe(3) -}, { retry: 3 }) +}, { retry: 2 }) let count2 = 0 it.fails('retry test fails', () => { count2 += 1 expect(count2).toBe(3) -}, { retry: 2 }) +}, { retry: 1 }) let count3 = 0 it('retry test fails', () => { diff --git a/test/failing/fixtures/expects/soft.test.ts b/test/failing/fixtures/expects/soft.test.ts index d7bf8c48744e..dd7d67afcfea 100644 --- a/test/failing/fixtures/expects/soft.test.ts +++ b/test/failing/fixtures/expects/soft.test.ts @@ -73,7 +73,7 @@ test('retry will passed', () => { expect.soft(num += 1).toBe(3) expect.soft(num += 1).toBe(4) }, { - retry: 2, + retry: 1, }) num = 0 @@ -81,5 +81,5 @@ test('retry will failed', () => { expect.soft(num += 1).toBe(4) expect.soft(num += 1).toBe(5) }, { - retry: 2, + retry: 1, })