diff --git a/docs/api/index.md b/docs/api/index.md index 259b47820a42..64fb061caca6 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -312,6 +312,23 @@ When you use `test` in the top level of file, they are collected as part of the describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */) ``` +### describe.shuffle + +- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` + + Vitest provides a way to run all tests in random order via CLI flag [`--sequence.shuffle`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. + + ```ts + describe.shuffle('suite', () => { + test('random test 1', async () => { /* ... */ }) + test('random test 2', async () => { /* ... */ }) + test('random test 3', async () => { /* ... */ }) + }) + // order depends on sequence.seed option in config (Date.now() by default) + ``` + +`.skip`, `.only`, and `.todo` works with random suites. + ### describe.todo - **Type:** `(name: string) => void` diff --git a/docs/config/index.md b/docs/config/index.md index cb8b8eacc7ac..a2850705c67f 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -582,3 +582,34 @@ Options to configure Vitest cache policy. At the moment Vitest stores cache for - **Default**: `node_modules/.vitest` Path to cache directory. + +### sequence + +- **Type**: `{ sequencer?, shuffle?, seed? }` + +Options for how tests should be sorted. + +#### sequence.sequencer + +- **Type**: `TestSequencerConstructor` +- **Default**: `BaseSequencer` + +A custom class that defines methods for sharding and sorting. You can extend `BaseSequencer` from `vitest/node`, if you only need to redefine one of the `sort` and `shard` methods, but both should exist. + +Sharding is happening before sorting, and only if `--shard` option is provided. + +#### sequence.shuffle + +- **Type**: `boolean` +- **Default**: `false` + +If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). + +Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. + +#### sequence.seed + +- **Type**: `number` +- **Default**: `Date.now()` + +Sets the randomization seed, if tests are running in random order. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e43fccc19775..614d06ad572a 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -36,10 +36,6 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit vitest related /src/index.ts /src/hello-world.js ``` -### `vitest clean cache` - -Clears cache folder. - ## Options | Options | | @@ -72,6 +68,7 @@ Clears cache folder. | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) | | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) | | `--shard ` | Execute tests in a specified shard | +| `--sequence` | Define in what order to run tests. Use [cac's dot notation] to specify options (for example, use `--sequence.suffle` to run tests in random order) | | `-h, --help` | Display available CLI options | ### changed diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 496f4ce88f28..a8002f2373f1 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -35,6 +35,7 @@ cli .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)') .option('--shard ', 'Test suite shard to execute in a format of /') .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') + .option('--sequence ', 'Define in what order to run tests (use --sequence.shuffle to run tests in random order)') .help() cli diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 7ec1791bcfb7..ec3212e0fbf2 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -9,6 +9,8 @@ import { configDefaults } from '../defaults' import { resolveC8Options } from '../integrations/coverage' import { toArray } from '../utils' import { VitestCache } from './cache' +import { BaseSequencer } from './sequencers/BaseSequencer' +import { RandomSequencer } from './sequencers/RandomSequencer' const extraInlineDeps = [ /^(?!.*(?:node_modules)).*\.mjs$/, @@ -186,5 +188,13 @@ export function resolveConfig( if (resolved.cache) resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir) + if (!resolved.sequence?.sequencer) { + resolved.sequence ??= {} as any + // CLI flag has higher priority + resolved.sequence.sequencer = resolved.sequence.shuffle + ? RandomSequencer + : BaseSequencer + } + return resolved } diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index d31b0c1e1505..0409ca5ef7f6 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -105,6 +105,10 @@ export class Vitest { resolveSnapshotPath: undefined, }, onConsoleLog: undefined!, + sequence: { + ...this.config.sequence, + sequencer: undefined!, + }, }, this.configOverride || {} as any, ) as ResolvedConfig diff --git a/packages/vitest/src/node/index.ts b/packages/vitest/src/node/index.ts index dd2952893e29..0fedd63e3a17 100644 --- a/packages/vitest/src/node/index.ts +++ b/packages/vitest/src/node/index.ts @@ -5,3 +5,6 @@ export { startVitest } from './cli-api' export { VitestRunner } from '../runtime/execute' export type { ExecuteOptions } from '../runtime/execute' + +export type { TestSequencer, TestSequencerContructor } from './sequencers/types' +export { BaseSequencer } from './sequencers/BaseSequencer' diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index bc1cb0b9492e..8dee562bc8cc 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -10,7 +10,6 @@ import type { ResolvedConfig, WorkerContext, WorkerRPC } from '../types' import { distDir } from '../constants' import { AggregateError } from '../utils' import type { Vitest } from './core' -import { BaseSequelizer } from './sequelizers/BaseSequelizer' export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise @@ -86,15 +85,16 @@ export function createPool(ctx: Vitest): WorkerPool { } } - const sequelizer = new BaseSequelizer(ctx) + const Sequencer = ctx.config.sequence.sequencer + const sequencer = new Sequencer(ctx) return async (files, invalidates) => { const config = ctx.getSerializableConfig() if (config.shard) - files = await sequelizer.shard(files) + files = await sequencer.shard(files) - files = await sequelizer.sort(files) + files = await sequencer.sort(files) if (!ctx.config.threads) { await runFiles(config, files) diff --git a/packages/vitest/src/node/sequelizers/BaseSequelizer.ts b/packages/vitest/src/node/sequencers/BaseSequencer.ts similarity index 94% rename from packages/vitest/src/node/sequelizers/BaseSequelizer.ts rename to packages/vitest/src/node/sequencers/BaseSequencer.ts index dc6a8ec9a4a9..ce0da56735dc 100644 --- a/packages/vitest/src/node/sequelizers/BaseSequelizer.ts +++ b/packages/vitest/src/node/sequencers/BaseSequencer.ts @@ -2,9 +2,9 @@ import { createHash } from 'crypto' import { resolve } from 'pathe' import { slash } from 'vite-node/utils' import type { Vitest } from '../core' -import type { TestSequelizer } from './types' +import type { TestSequencer } from './types' -export class BaseSequelizer implements TestSequelizer { +export class BaseSequencer implements TestSequencer { protected ctx: Vitest constructor(ctx: Vitest) { diff --git a/packages/vitest/src/node/sequencers/RandomSequencer.ts b/packages/vitest/src/node/sequencers/RandomSequencer.ts new file mode 100644 index 000000000000..c92c82f62080 --- /dev/null +++ b/packages/vitest/src/node/sequencers/RandomSequencer.ts @@ -0,0 +1,12 @@ +import { shuffle } from '../../utils' +import { BaseSequencer } from './BaseSequencer' + +export class RandomSequencer extends BaseSequencer { + public async sort(files: string[]) { + const { sequence } = this.ctx.config + + const seed = sequence?.seed ?? Date.now() + + return shuffle(files, seed) + } +} diff --git a/packages/vitest/src/node/sequelizers/types.ts b/packages/vitest/src/node/sequencers/types.ts similarity index 71% rename from packages/vitest/src/node/sequelizers/types.ts rename to packages/vitest/src/node/sequencers/types.ts index 5442d0a1c4b2..40344d24d0ae 100644 --- a/packages/vitest/src/node/sequelizers/types.ts +++ b/packages/vitest/src/node/sequencers/types.ts @@ -1,7 +1,7 @@ import type { Awaitable } from '../../types' import type { Vitest } from '../core' -export interface TestSequelizer { +export interface TestSequencer { /** * Slicing tests into shards. Will be run before `sort`. * Only run, if `shard` is defined. @@ -10,6 +10,6 @@ export interface TestSequelizer { sort(files: string[]): Awaitable } -export interface TestSequelizerContructor { - new (ctx: Vitest): TestSequelizer +export interface TestSequencerContructor { + new (ctx: Vitest): TestSequencer } diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index b3cd233110e6..d6bc51fc2830 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -2,7 +2,7 @@ import limit from 'p-limit' import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' import { vi } from '../integrations/vi' import { getSnapshotClient } from '../integrations/snapshot/chai' -import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout } from '../utils' +import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout, shuffle } from '../utils' import { takeCoverage } from '../integrations/coverage' import { getState, setState } from '../integrations/chai/jest-expect' import { GLOBAL_EXPECT } from '../integrations/chai/constants' @@ -210,12 +210,20 @@ export async function runSuite(suite: Suite) { try { const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite]) - for (const tasksGroup of partitionSuiteChildren(suite)) { + for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { const mutex = limit(workerState.config.maxConcurrency) await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c)))) } else { + const { sequence } = workerState.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + } for (const c of tasksGroup) await runSuiteChild(c) } diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 99d8b394339e..e3f464f20085 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -1,6 +1,6 @@ import { format } from 'util' import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types' -import { isObject, noop } from '../utils' +import { getWorkerState, isObject, noop } from '../utils' import { createChainable } from './chain' import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context' import { getHooks, setFn, setHooks } from './map' @@ -37,8 +37,12 @@ function formatTitle(template: string, items: any[], idx: number) { export const describe = suite export const it = test +const workerState = getWorkerState() + // implementations -export const defaultSuite = suite('') +export const defaultSuite = workerState.config.sequence.shuffle + ? suite.shuffle('') + : suite('') export function clearCollectorContext() { collectorContext.tasks.length = 0 @@ -59,7 +63,7 @@ export function createSuiteHooks() { } } -function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean) { +function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, shuffle?: boolean) { const tasks: (Test | Suite | SuiteCollector)[] = [] const factoryQueue: (Test | Suite | SuiteCollector)[] = [] @@ -80,6 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } as Omit as Test if (this.concurrent || concurrent) test.concurrent = true + if (shuffle) + test.shuffle = true const context = createTestContext(test) // create test context @@ -117,6 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m type: 'suite', name, mode, + shuffle, tasks: [], } setHooks(suite, createSuiteHooks()) @@ -157,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { const suite = createChainable( - ['concurrent', 'skip', 'only', 'todo'], + ['concurrent', 'shuffle', 'skip', 'only', 'todo'], function (name: string, factory?: SuiteFactory) { const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' - return createSuiteCollector(name, factory, mode, this.concurrent) + return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle) }, ) as SuiteAPI diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index db7f48ccce82..9f0bb9ebd44e 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -2,6 +2,7 @@ import type { CommonServerOptions } from 'vite' import type { PrettyFormatOptions } from 'pretty-format' import type { FakeTimerInstallOpts } from '@sinonjs/fake-timers' import type { BuiltinReporters } from '../node/reporters' +import type { TestSequencerContructor } from '../node/sequencers/types' import type { C8Options, ResolvedC8Options } from './coverage' import type { JSDOMOptions } from './jsdom-options' import type { Reporter } from './reporter' @@ -369,6 +370,29 @@ export interface InlineConfig { cache?: false | { dir?: string } + + /** + * Options for configuring the order of running tests. + */ + sequence?: { + /** + * Class that handles sorting and sharding algorithm. + * If you only need to change sorting, you can extend + * your custom sequencer from `BaseSequencer` from `vitest/node`. + * @default BaseSequencer + */ + sequencer?: TestSequencerContructor + /** + * Should tests run in random order. + * @default false + */ + shuffle?: boolean + /** + * Seed for the random number generator. + * @default Date.now() + */ + seed?: number + } } export interface UserConfig extends InlineConfig { @@ -415,7 +439,7 @@ export interface UserConfig extends InlineConfig { shard?: string } -export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache'> { +export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { base?: string config?: string @@ -439,4 +463,10 @@ export interface ResolvedConfig extends Omit, 'config' | 'f cache: { dir: string } | false + + sequence: { + sequencer: TestSequencerContructor + shuffle?: boolean + seed?: number + } } diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index aaa91746bfc9..773601590d78 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -10,6 +10,7 @@ export interface TaskBase { name: string mode: RunMode concurrent?: boolean + shuffle?: boolean suite?: Suite file?: File result?: TaskResult @@ -113,7 +114,7 @@ void } export type SuiteAPI = ChainableFunction< -'concurrent' | 'only' | 'skip' | 'todo', +'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle', [name: string, factory?: SuiteFactory], SuiteCollector > & { diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index f872c3d74372..9633c670cfe0 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -1,3 +1,4 @@ +import { RealDate } from '../integrations/mock/date' import type { Arrayable, DeepMerge, Nullable } from '../types' function isFinalObj(obj: any) { @@ -153,3 +154,23 @@ export function stdout(): NodeJS.WriteStream { // eslint-disable-next-line no-console return console._stdout || process.stdout } + +function random(seed: number) { + const x = Math.sin(seed++) * 10000 + return x - Math.floor(x) +} + +export function shuffle(array: T[], seed = RealDate.now()): T[] { + let length = array.length + + while (length) { + const index = Math.floor(random(seed) * length--) + + const previous = array[length] + array[length] = array[index] + array[index] = previous + ++seed + } + + return array +} diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts new file mode 100644 index 000000000000..fefc9594b812 --- /dev/null +++ b/test/core/test/random.test.ts @@ -0,0 +1,31 @@ +import { afterAll, describe, expect, test } from 'vitest' + +// tests use seed of 101, so they have deterministic random order +const numbers: number[] = [] + +describe.shuffle('random tests', () => { + describe('inside', () => { + // shuffle is not inhereted from parent + + test('inside 1', () => { + numbers.push(1) + }) + test('inside 2', () => { + numbers.push(2) + }) + }) + + test('test 1', () => { + numbers.push(3) + }) + test('test 2', () => { + numbers.push(4) + }) + test('test 3', () => { + numbers.push(5) + }) + + afterAll(() => { + expect(numbers).toStrictEqual([4, 5, 3, 1, 2]) + }) +}) diff --git a/test/core/test/sequelizers.test.ts b/test/core/test/sequencers.test.ts similarity index 68% rename from test/core/test/sequelizers.test.ts rename to test/core/test/sequencers.test.ts index 1980f266a709..745ae8e7da03 100644 --- a/test/core/test/sequelizers.test.ts +++ b/test/core/test/sequencers.test.ts @@ -1,9 +1,13 @@ import type { Vitest } from 'vitest' import { describe, expect, test, vi } from 'vitest' -import { BaseSequelizer } from '../../../packages/vitest/src/node/sequelizers/BaseSequelizer' +import { RandomSequencer } from 'vitest/src/node/sequencers/RandomSequencer' +import { BaseSequencer } from 'vitest/src/node/sequencers/BaseSequencer' const buildCtx = () => { return { + config: { + sequence: {}, + }, state: { getFileTestResults: vi.fn(), getFileStats: vi.fn(), @@ -11,11 +15,11 @@ const buildCtx = () => { } as unknown as Vitest } -describe('test sequelizers', () => { +describe('base sequencer', () => { test('sorting when no info is available', async () => { - const sequelizer = new BaseSequelizer(buildCtx()) + const sequencer = new BaseSequencer(buildCtx()) const files = ['a', 'b', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(files) }) @@ -25,9 +29,9 @@ describe('test sequelizers', () => { if (file === 'b') return { size: 2 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['a', 'c', 'b']) }) @@ -41,9 +45,9 @@ describe('test sequelizers', () => { if (file === 'c') return { size: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -57,9 +61,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 1 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['b', 'c', 'a']) }) @@ -73,9 +77,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -89,9 +93,20 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) }) + +describe('random sequencer', () => { + test('sorting is the same when seed is defined', async () => { + const ctx = buildCtx() + ctx.config.sequence.seed = 101 + const sequencer = new RandomSequencer(ctx) + const files = ['b', 'a', 'c'] + const sorted = await sequencer.sort(files) + expect(sorted).toStrictEqual(['a', 'c', 'b']) + }) +}) diff --git a/test/core/vitest.config.ts b/test/core/vitest.config.ts index e6ca56714997..4b41690a8289 100644 --- a/test/core/vitest.config.ts +++ b/test/core/vitest.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ return path + extension return join(dirname(path), '__snapshots__', `${basename(path)}${extension}`) }, + sequence: { + seed: 101, + }, }, })