Skip to content

Commit

Permalink
Changing repeat and retry logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@samkevin1
samkevin1 committed Mar 8, 2023
1 parent 47ff3d5 commit f706072
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 122 deletions.
14 changes: 1 addition & 13 deletions docs/api/index.md
View file
Expand Up @@ -293,19 +293,7 @@ You cannot use this syntax, when using Vitest as [type checker](/guide/testing-t
- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void`
- **Alias:** `it.repeats`

If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so.

By default it will repeat 5 times:

```ts
import { expect, test } from 'vitest'

test.repeats('repeated test', () => {
expect(true).toBe(true)
})
```

To change the default `repeats` value:
If you want to run a test multiple times to see if it passes on all attempts, you can use `test.repeats` to do so. Without the `repeats` options it will only run once.

```ts
import { expect, test } from 'vitest'
Expand Down
201 changes: 105 additions & 96 deletions packages/runner/src/run.ts
View file
Expand Up @@ -125,53 +125,58 @@ export async function runTest(test: Test, runner: VitestRunner) {

setCurrentTest(test)

const retry = test.mode === 'repeats' ? test.repeats! : test.retry || 1
for (let retryCount = 0; retryCount < retry; retryCount++) {
let beforeEachCleanups: HookCleanupCallback[] = []
try {
await runner.onBeforeTryTest?.(test, retryCount)
const repeats = test.mode === 'repeats' ? test.repeats || 1 : 1

beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
const retry = test.retry || 1

test.result.retryCount = retryCount
for (let retryCount = 0; retryCount < retry; retryCount++) {
let beforeEachCleanups: HookCleanupCallback[] = []
try {
await runner.onBeforeTryTest?.(test, retryCount)

if (runner.runTest) {
await runner.runTest(test)
}
else {
const fn = getFn(test)
if (!fn)
throw new Error('Test function is not found. Did you add it using `setFn`?')
await fn()
}
beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])

await runner.onAfterTryTest?.(test, retryCount)
test.result.retryCount = retryCount

if (test.mode === 'run')
test.result.state = 'pass'
else if (test.mode === 'repeats' && retry === retryCount)
test.result.state = 'pass'
}
catch (e) {
failTask(test.result, e)
}
if (runner.runTest) {
await runner.runTest(test)
}
else {
const fn = getFn(test)
if (!fn)
throw new Error('Test function is not found. Did you add it using `setFn`?')
await fn()
}

try {
await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
await callCleanupHooks(beforeEachCleanups)
}
catch (e) {
failTask(test.result, e)
}
await runner.onAfterTryTest?.(test, retryCount)

if (test.mode === 'run')
test.result.state = 'pass'
else if (test.mode === 'repeats' && retry === retryCount)
test.result.state = 'pass'
}
catch (e) {
failTask(test.result, e)
}

if (test.result.state === 'pass')
break
try {
await callSuiteHook(test.suite, test, 'afterEach', runner, [test.context, test.suite])
await callCleanupHooks(beforeEachCleanups)
}
catch (e) {
failTask(test.result, e)
}

if (test.mode === 'repeats' && test.result.state === 'fail')
break
if (test.result.state === 'pass')
break

// update retry info
updateTask(test, runner)
if (test.mode === 'repeats' && retry === retryCount && test.result.state === 'fail')
break

// update retry info
updateTask(test, runner)
}
}

if (test.result.state === 'fail')
Expand Down Expand Up @@ -246,79 +251,83 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
suite.result.state = 'todo'
}
else {
const retry = suite.mode === 'repeats' ? suite.repeats! : suite.retry || 1
const repeats = suite.mode === 'repeats' ? suite.repeats || 1 : 1

for (let retryCount = 0; retryCount < retry; retryCount++) {
try {
beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite])
for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
const retry = suite.retry || 1

if (runner.runSuite) {
await runner.runSuite(suite)
}
else {
for (let tasksGroup of partitionSuiteChildren(suite)) {
if (tasksGroup[0].concurrent === true) {
const mutex = limit(runner.config.maxConcurrency)
await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
}
else {
const { sequence } = runner.config
if (sequence.shuffle || suite.shuffle) {
// run describe block independently from tests
const suites = tasksGroup.filter(group => group.type === 'suite')
const tests = tasksGroup.filter(group => group.type === 'test')
const groups = shuffle([suites, tests], sequence.seed)
tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed))
for (let retryCount = 0; retryCount < retry; retryCount++) {
try {
beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', runner, [suite])

if (runner.runSuite) {
await runner.runSuite(suite)
}
else {
for (let tasksGroup of partitionSuiteChildren(suite)) {
if (tasksGroup[0].concurrent === true) {
const mutex = limit(runner.config.maxConcurrency)
await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
}
else {
const { sequence } = runner.config
if (sequence.shuffle || suite.shuffle) {
// run describe block independently from tests
const suites = tasksGroup.filter(group => group.type === 'suite')
const tests = tasksGroup.filter(group => group.type === 'test')
const groups = shuffle([suites, tests], sequence.seed)
tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed))
}
for (const c of tasksGroup)
await runSuiteChild(c, runner)
}
for (const c of tasksGroup)
await runSuiteChild(c, runner)
}
}
}
}
catch (e) {
failTask(suite.result, e)
}

try {
if (suite.mode !== 'repeats')
await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
else if (suite.mode === 'repeats' && retry === retryCount)
await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
await callCleanupHooks(beforeAllCleanups)
}
catch (e) {
failTask(suite.result, e)
}
catch (e) {
failTask(suite.result, e)
}

if (suite.mode === 'run') {
if (!hasTests(suite)) {
suite.result.state = 'fail'
if (!suite.result.error) {
const error = processError(new Error(`No test found in suite ${suite.name}`))
suite.result.error = error
suite.result.errors = [error]
}
try {
if (suite.mode !== 'repeats')
await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
else if (suite.mode === 'repeats' && repeatCount - 1 === repeats && retry === retryCount)
await callSuiteHook(suite, suite, 'afterAll', runner, [suite])
await callCleanupHooks(beforeAllCleanups)
}
else if (hasFailed(suite)) {
suite.result.state = 'fail'
catch (e) {
failTask(suite.result, e)
}
else {
suite.result.state = 'pass'

if (suite.mode === 'run') {
if (!hasTests(suite)) {
suite.result.state = 'fail'
if (!suite.result.error) {
const error = processError(new Error(`No test found in suite ${suite.name}`))
suite.result.error = error
suite.result.errors = [error]
}
}
else if (hasFailed(suite)) {
suite.result.state = 'fail'
}
else {
suite.result.state = 'pass'
}
}
}

updateTask(suite, runner)
updateTask(suite, runner)

suite.result.duration = now() - start
suite.result.duration = now() - start

await runner.onAfterRunSuite?.(suite)
await runner.onAfterRunSuite?.(suite)

if (suite.result.state === 'pass')
break
if (suite.result.state === 'pass')
break

if (suite.mode === 'repeats' && suite.result.state === 'fail')
break
if (suite.mode === 'repeats' && suite.result.state === 'fail')
break
}
}
}
}
Expand Down
5 changes: 2 additions & 3 deletions packages/runner/src/suite.ts
View file
Expand Up @@ -73,8 +73,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
suite: undefined!,
fails: this.fails,
retry: options?.retry,
// 5 repetitions by default
repeats: mode === 'repeats' && !options?.repeats ? 5 : options?.repeats,
repeats: options?.repeats,
} as Omit<Test, 'context'> as Test

if (this.concurrent || concurrent)
Expand Down Expand Up @@ -136,7 +135,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
mode,
shuffle,
tasks: [],
repeats: mode === 'repeats' && !suiteOptions?.repeats ? 5 : suiteOptions?.repeats,
repeats: suiteOptions?.repeats,
}

setHooks(suite, createSuiteHooks())
Expand Down
27 changes: 17 additions & 10 deletions test/core/test/repeats.test.ts
View file
Expand Up @@ -4,10 +4,10 @@ const testNumbers: number[] = []

describe('testing it/test', () => {
const result = [1, 1, 1, 1, 1, 2, 2, 2]
// repeats 5 times by default

test.repeats('test 1', () => {
testNumbers.push(1)
})
}, { repeats: 5 })

test.repeats('test 2', () => {
testNumbers.push(2)
Expand All @@ -26,18 +26,25 @@ describe('testing it/test', () => {

const describeNumbers: number[] = []

describe.repeats('testing describe 1', () => {
describe.repeats('testing describe', () => {
test('test 1', () => {
describeNumbers.push(1)
})
})

describe.repeats('testing describe 2', () => {
test('test 2', () => {
describeNumbers.push(2)
})
}, { repeats: 3 })

afterAll(() => {
expect(describeNumbers).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2])
expect(describeNumbers).toStrictEqual([1, 1, 1])
})

const retryNumbers: number[] = []

describe('testing repeats with retry', () => {
const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
test.repeats('test 1', () => {
retryNumbers.push(1)
}, { repeats: 5, retry: 2 })

afterAll(() => {
expect(retryNumbers).toStrictEqual(result)
})
})

0 comments on commit f706072

Please sign in to comment.