diff --git a/docs/api/index.md b/docs/api/index.md index 259b47820a42..9b84d42172ab 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -312,6 +312,23 @@ When you use `test` in the top level of file, they are collected as part of the describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */) ``` +### describe.random + +- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` + + Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.random`](/config/#sequence-random), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. + + ```ts + describe.random('suite', () => { + test('concurrent test 1', async () => { /* ... */ }) + test('concurrent test 2', async () => { /* ... */ }) + test('concurrent test 3', async () => { /* ... */ }) + }) + // order depends on sequence.seed option in config (Date.now() by default) + ``` + +`.skip`, `.only`, and `.todo` works with random suites. + ### describe.todo - **Type:** `(name: string) => void` diff --git a/docs/config/index.md b/docs/config/index.md index cb8b8eacc7ac..fbf90e33d3dc 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -582,3 +582,34 @@ Options to configure Vitest cache policy. At the moment Vitest stores cache for - **Default**: `node_modules/.vitest` Path to cache directory. + +### sequence + +- **Type**: `{ sequencer?, random?, seed? }` + +Options for how tests should be sorted. + +#### sequence.sequencer + +- **Type**: `TestSequencerConstructor` +- **Default**: `BaseSequencer` + +A custom class that defines methods for sharding and sorting. You can extend `BaseSequencer` from `vitest/node`, if you only need to redefine one of the `sort` and `shard` methods, but both should exist. + +Sharding is happening before sorting, and only if `--shard` option is provided. + +#### sequence.random + +- **Type**: `boolean` +- **Default**: `false` + +If you want tests to run randomly, you can enable it with this option, or CLI argument [`--random`](/guide/cli). + +Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. + +#### sequence.seed + +- **Type**: `number` +- **Default**: `Date.now()` + +Sets the randomization seed, if tests are running in random order. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e43fccc19775..62a52d5665c0 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -36,10 +36,6 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit vitest related /src/index.ts /src/hello-world.js ``` -### `vitest clean cache` - -Clears cache folder. - ## Options | Options | | @@ -72,6 +68,7 @@ Clears cache folder. | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) | | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) | | `--shard ` | Execute tests in a specified shard | +| `--random` | Execute tests in random order | | `-h, --help` | Display available CLI options | ### changed diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 496f4ce88f28..d2dc1087383e 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -35,6 +35,7 @@ cli .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)') .option('--shard ', 'Test suite shard to execute in a format of /') .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') + .option('--random', 'Run tests in random order (default: false)') .help() cli diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 3df329b8f8fb..87087d31a056 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -191,6 +191,7 @@ export function resolveConfig( // random should have the priority over config, because it is a CLI flag if (!resolved.sequence?.sequencer || resolved.random) { resolved.sequence ??= {} as any + resolved.sequence.random ??= resolved.random resolved.sequence.sequencer = resolved.sequence.random || resolved.random ? RandomSequencer : BaseSequencer diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 8db21b680b5b..0409ca5ef7f6 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -105,7 +105,10 @@ export class Vitest { resolveSnapshotPath: undefined, }, onConsoleLog: undefined!, - sequence: undefined!, + sequence: { + ...this.config.sequence, + sequencer: undefined!, + }, }, this.configOverride || {} as any, ) as ResolvedConfig diff --git a/packages/vitest/src/node/sequencers/RandomSequencer.ts b/packages/vitest/src/node/sequencers/RandomSequencer.ts index b7c8a569e34a..87ac84852353 100644 --- a/packages/vitest/src/node/sequencers/RandomSequencer.ts +++ b/packages/vitest/src/node/sequencers/RandomSequencer.ts @@ -1,26 +1,12 @@ +import { randomize } from '../../utils' import { BaseSequencer } from './BaseSequencer' export class RandomSequencer extends BaseSequencer { - private random(seed: number) { - const x = Math.sin(seed++) * 10000 - return x - Math.floor(x) - } - public async sort(files: string[]) { const { sequence } = this.ctx.config - let seed = sequence?.seed ?? Date.now() - let length = files.length - - while (length) { - const index = Math.floor(this.random(seed) * length--) - - const previous = files[length] - files[length] = files[index] - files[index] = previous - ++seed - } + const seed = sequence?.seed ?? Date.now() - return files + return randomize(files, seed) } } diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index b3cd233110e6..bcc11ef9c65e 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -2,7 +2,7 @@ import limit from 'p-limit' import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' import { vi } from '../integrations/vi' import { getSnapshotClient } from '../integrations/snapshot/chai' -import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout } from '../utils' +import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, randomize, setTimeout } from '../utils' import { takeCoverage } from '../integrations/coverage' import { getState, setState } from '../integrations/chai/jest-expect' import { GLOBAL_EXPECT } from '../integrations/chai/constants' @@ -210,12 +210,20 @@ export async function runSuite(suite: Suite) { try { const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite]) - for (const tasksGroup of partitionSuiteChildren(suite)) { + for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { const mutex = limit(workerState.config.maxConcurrency) await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c)))) } else { + const { sequence } = workerState.config + if (sequence.random || suite.random) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = randomize([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => randomize(group, sequence.seed)) + } for (const c of tasksGroup) await runSuiteChild(c) } diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 99d8b394339e..35113aec0712 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -1,6 +1,6 @@ import { format } from 'util' import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types' -import { isObject, noop } from '../utils' +import { getWorkerState, isObject, noop } from '../utils' import { createChainable } from './chain' import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context' import { getHooks, setFn, setHooks } from './map' @@ -37,8 +37,12 @@ function formatTitle(template: string, items: any[], idx: number) { export const describe = suite export const it = test +const workerState = getWorkerState() + // implementations -export const defaultSuite = suite('') +export const defaultSuite = workerState.config.sequence.random + ? suite.random('') + : suite('') export function clearCollectorContext() { collectorContext.tasks.length = 0 @@ -59,7 +63,7 @@ export function createSuiteHooks() { } } -function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean) { +function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, random?: boolean) { const tasks: (Test | Suite | SuiteCollector)[] = [] const factoryQueue: (Test | Suite | SuiteCollector)[] = [] @@ -80,6 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } as Omit as Test if (this.concurrent || concurrent) test.concurrent = true + if (random) + test.random = true const context = createTestContext(test) // create test context @@ -117,6 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m type: 'suite', name, mode, + random, tasks: [], } setHooks(suite, createSuiteHooks()) @@ -157,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { const suite = createChainable( - ['concurrent', 'skip', 'only', 'todo'], + ['concurrent', 'random', 'skip', 'only', 'todo'], function (name: string, factory?: SuiteFactory) { const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' - return createSuiteCollector(name, factory, mode, this.concurrent) + return createSuiteCollector(name, factory, mode, this.concurrent, this.random) }, ) as SuiteAPI diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 5ebcf13849db..08370a8c9405 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -442,7 +442,7 @@ export interface UserConfig extends InlineConfig { * If tests should be run in random. * @default false */ - random?: string + random?: boolean } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index aaa91746bfc9..afc78ef83e38 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -10,6 +10,7 @@ export interface TaskBase { name: string mode: RunMode concurrent?: boolean + random?: boolean suite?: Suite file?: File result?: TaskResult @@ -113,7 +114,7 @@ void } export type SuiteAPI = ChainableFunction< -'concurrent' | 'only' | 'skip' | 'todo', +'concurrent' | 'only' | 'skip' | 'todo' | 'random', [name: string, factory?: SuiteFactory], SuiteCollector > & { diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index f872c3d74372..931e4f65769d 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -153,3 +153,24 @@ export function stdout(): NodeJS.WriteStream { // eslint-disable-next-line no-console return console._stdout || process.stdout } + +function random(seed: number) { + const x = Math.sin(seed++) * 10000 + return x - Math.floor(x) +} + +export function randomize(array: T[], seed?: number): T[] { + let length = array.length + seed ??= Date.now() + + while (length) { + const index = Math.floor(random(seed) * length--) + + const previous = array[length] + array[length] = array[index] + array[index] = previous + ++seed + } + + return array +} diff --git a/test/core/package.json b/test/core/package.json index 04d6e0b02701..aee453e7d942 100644 --- a/test/core/package.json +++ b/test/core/package.json @@ -2,7 +2,7 @@ "name": "@vitest/test-core", "private": true, "scripts": { - "test": "vitest", + "test": "vitest test/core/test/random.test.ts", "coverage": "vitest run --coverage" }, "devDependencies": { diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts new file mode 100644 index 000000000000..dc9284ba635f --- /dev/null +++ b/test/core/test/random.test.ts @@ -0,0 +1,31 @@ +import { afterAll, describe, expect, test } from 'vitest' + +// tests use seed of 101, so they have deterministic random order +const numbers: number[] = [] + +describe.random('random tests', () => { + describe('inside', () => { + // random is not inhereted from parent + + test('inside 1', () => { + numbers.push(1) + }) + test('inside 2', () => { + numbers.push(2) + }) + }) + + test('test 1', () => { + numbers.push(3) + }) + test('test 2', () => { + numbers.push(4) + }) + test('test 3', () => { + numbers.push(5) + }) + + afterAll(() => { + expect(numbers).toStrictEqual([4, 5, 3, 1, 2]) + }) +}) diff --git a/test/core/test/sequencers.test.ts b/test/core/test/sequencers.test.ts index 5b1b84649ea9..745ae8e7da03 100644 --- a/test/core/test/sequencers.test.ts +++ b/test/core/test/sequencers.test.ts @@ -1,9 +1,13 @@ import type { Vitest } from 'vitest' import { describe, expect, test, vi } from 'vitest' -import { BaseSequencer } from '../../../packages/vitest/src/node/sequencers/BaseSequencer' +import { RandomSequencer } from 'vitest/src/node/sequencers/RandomSequencer' +import { BaseSequencer } from 'vitest/src/node/sequencers/BaseSequencer' const buildCtx = () => { return { + config: { + sequence: {}, + }, state: { getFileTestResults: vi.fn(), getFileStats: vi.fn(), @@ -11,11 +15,11 @@ const buildCtx = () => { } as unknown as Vitest } -describe('test sequelizers', () => { +describe('base sequencer', () => { test('sorting when no info is available', async () => { - const sequelizer = new BaseSequencer(buildCtx()) + const sequencer = new BaseSequencer(buildCtx()) const files = ['a', 'b', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(files) }) @@ -25,9 +29,9 @@ describe('test sequelizers', () => { if (file === 'b') return { size: 2 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['a', 'c', 'b']) }) @@ -41,9 +45,9 @@ describe('test sequelizers', () => { if (file === 'c') return { size: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -57,9 +61,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 1 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['b', 'c', 'a']) }) @@ -73,9 +77,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -89,9 +93,20 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) }) + +describe('random sequencer', () => { + test('sorting is the same when seed is defined', async () => { + const ctx = buildCtx() + ctx.config.sequence.seed = 101 + const sequencer = new RandomSequencer(ctx) + const files = ['b', 'a', 'c'] + const sorted = await sequencer.sort(files) + expect(sorted).toStrictEqual(['a', 'c', 'b']) + }) +}) diff --git a/test/core/vitest.config.ts b/test/core/vitest.config.ts index e6ca56714997..4b41690a8289 100644 --- a/test/core/vitest.config.ts +++ b/test/core/vitest.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ return path + extension return join(dirname(path), '__snapshots__', `${basename(path)}${extension}`) }, + sequence: { + seed: 101, + }, }, })