From 6205f9e77dfc68641420a93ac09b2d42bc7d8d27 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Sun, 3 Jul 2022 15:46:36 +0300 Subject: [PATCH 01/11] feat: make sequencer public, add random option --- packages/vitest/src/node/config.ts | 10 +++++ packages/vitest/src/node/core.ts | 1 + packages/vitest/src/node/index.ts | 3 ++ packages/vitest/src/node/pool.ts | 8 ++-- .../BaseSequencer.ts} | 4 +- .../src/node/sequencers/RandomSequencer.ts | 26 +++++++++++++ .../node/{sequelizers => sequencers}/types.ts | 6 +-- packages/vitest/src/types/config.ts | 38 ++++++++++++++++++- 8 files changed, 86 insertions(+), 10 deletions(-) rename packages/vitest/src/node/{sequelizers/BaseSequelizer.ts => sequencers/BaseSequencer.ts} (94%) create mode 100644 packages/vitest/src/node/sequencers/RandomSequencer.ts rename packages/vitest/src/node/{sequelizers => sequencers}/types.ts (71%) diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 7ec1791bcfb7..3df329b8f8fb 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -9,6 +9,8 @@ import { configDefaults } from '../defaults' import { resolveC8Options } from '../integrations/coverage' import { toArray } from '../utils' import { VitestCache } from './cache' +import { BaseSequencer } from './sequencers/BaseSequencer' +import { RandomSequencer } from './sequencers/RandomSequencer' const extraInlineDeps = [ /^(?!.*(?:node_modules)).*\.mjs$/, @@ -186,5 +188,13 @@ export function resolveConfig( if (resolved.cache) resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir) + // random should have the priority over config, because it is a CLI flag + if (!resolved.sequence?.sequencer || resolved.random) { + resolved.sequence ??= {} as any + resolved.sequence.sequencer = resolved.sequence.random || resolved.random + ? RandomSequencer + : BaseSequencer + } + return resolved } diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index d31b0c1e1505..8db21b680b5b 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -105,6 +105,7 @@ export class Vitest { resolveSnapshotPath: undefined, }, onConsoleLog: undefined!, + sequence: undefined!, }, this.configOverride || {} as any, ) as ResolvedConfig diff --git a/packages/vitest/src/node/index.ts b/packages/vitest/src/node/index.ts index dd2952893e29..0fedd63e3a17 100644 --- a/packages/vitest/src/node/index.ts +++ b/packages/vitest/src/node/index.ts @@ -5,3 +5,6 @@ export { startVitest } from './cli-api' export { VitestRunner } from '../runtime/execute' export type { ExecuteOptions } from '../runtime/execute' + +export type { TestSequencer, TestSequencerContructor } from './sequencers/types' +export { BaseSequencer } from './sequencers/BaseSequencer' diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index bc1cb0b9492e..8dee562bc8cc 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -10,7 +10,6 @@ import type { ResolvedConfig, WorkerContext, WorkerRPC } from '../types' import { distDir } from '../constants' import { AggregateError } from '../utils' import type { Vitest } from './core' -import { BaseSequelizer } from './sequelizers/BaseSequelizer' export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise @@ -86,15 +85,16 @@ export function createPool(ctx: Vitest): WorkerPool { } } - const sequelizer = new BaseSequelizer(ctx) + const Sequencer = ctx.config.sequence.sequencer + const sequencer = new Sequencer(ctx) return async (files, invalidates) => { const config = ctx.getSerializableConfig() if (config.shard) - files = await sequelizer.shard(files) + files = await sequencer.shard(files) - files = await sequelizer.sort(files) + files = await sequencer.sort(files) if (!ctx.config.threads) { await runFiles(config, files) diff --git a/packages/vitest/src/node/sequelizers/BaseSequelizer.ts b/packages/vitest/src/node/sequencers/BaseSequencer.ts similarity index 94% rename from packages/vitest/src/node/sequelizers/BaseSequelizer.ts rename to packages/vitest/src/node/sequencers/BaseSequencer.ts index dc6a8ec9a4a9..ce0da56735dc 100644 --- a/packages/vitest/src/node/sequelizers/BaseSequelizer.ts +++ b/packages/vitest/src/node/sequencers/BaseSequencer.ts @@ -2,9 +2,9 @@ import { createHash } from 'crypto' import { resolve } from 'pathe' import { slash } from 'vite-node/utils' import type { Vitest } from '../core' -import type { TestSequelizer } from './types' +import type { TestSequencer } from './types' -export class BaseSequelizer implements TestSequelizer { +export class BaseSequencer implements TestSequencer { protected ctx: Vitest constructor(ctx: Vitest) { diff --git a/packages/vitest/src/node/sequencers/RandomSequencer.ts b/packages/vitest/src/node/sequencers/RandomSequencer.ts new file mode 100644 index 000000000000..b7c8a569e34a --- /dev/null +++ b/packages/vitest/src/node/sequencers/RandomSequencer.ts @@ -0,0 +1,26 @@ +import { BaseSequencer } from './BaseSequencer' + +export class RandomSequencer extends BaseSequencer { + private random(seed: number) { + const x = Math.sin(seed++) * 10000 + return x - Math.floor(x) + } + + public async sort(files: string[]) { + const { sequence } = this.ctx.config + + let seed = sequence?.seed ?? Date.now() + let length = files.length + + while (length) { + const index = Math.floor(this.random(seed) * length--) + + const previous = files[length] + files[length] = files[index] + files[index] = previous + ++seed + } + + return files + } +} diff --git a/packages/vitest/src/node/sequelizers/types.ts b/packages/vitest/src/node/sequencers/types.ts similarity index 71% rename from packages/vitest/src/node/sequelizers/types.ts rename to packages/vitest/src/node/sequencers/types.ts index 5442d0a1c4b2..40344d24d0ae 100644 --- a/packages/vitest/src/node/sequelizers/types.ts +++ b/packages/vitest/src/node/sequencers/types.ts @@ -1,7 +1,7 @@ import type { Awaitable } from '../../types' import type { Vitest } from '../core' -export interface TestSequelizer { +export interface TestSequencer { /** * Slicing tests into shards. Will be run before `sort`. * Only run, if `shard` is defined. @@ -10,6 +10,6 @@ export interface TestSequelizer { sort(files: string[]): Awaitable } -export interface TestSequelizerContructor { - new (ctx: Vitest): TestSequelizer +export interface TestSequencerContructor { + new (ctx: Vitest): TestSequencer } diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index db7f48ccce82..5ebcf13849db 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -2,6 +2,7 @@ import type { CommonServerOptions } from 'vite' import type { PrettyFormatOptions } from 'pretty-format' import type { FakeTimerInstallOpts } from '@sinonjs/fake-timers' import type { BuiltinReporters } from '../node/reporters' +import type { TestSequencerContructor } from '../node/sequencers/types' import type { C8Options, ResolvedC8Options } from './coverage' import type { JSDOMOptions } from './jsdom-options' import type { Reporter } from './reporter' @@ -369,6 +370,29 @@ export interface InlineConfig { cache?: false | { dir?: string } + + /** + * Options for configuring the order of running tests. + */ + sequence?: { + /** + * Class that handles sorting and sharding algorithm. + * If you only need to change sorting, you can extend + * your custom sequencer from `BaseSequencer` from `vitest/node`. + * @default BaseSequencer + */ + sequencer?: TestSequencerContructor + /** + * Should tests run in random order. + * @default false + */ + random?: boolean + /** + * Seed for the random number generator. + * @default Date.now() + */ + seed?: number + } } export interface UserConfig extends InlineConfig { @@ -413,9 +437,15 @@ export interface UserConfig extends InlineConfig { * @example --shard=2/3 */ shard?: string + + /** + * If tests should be run in random. + * @default false + */ + random?: string } -export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache'> { +export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { base?: string config?: string @@ -439,4 +469,10 @@ export interface ResolvedConfig extends Omit, 'config' | 'f cache: { dir: string } | false + + sequence: { + sequencer: TestSequencerContructor + random?: boolean + seed?: number + } } From 7fe79c24bb307dd85aac5d53673d5596904e9804 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Sun, 3 Jul 2022 15:55:24 +0300 Subject: [PATCH 02/11] test: cleanup --- .../{sequelizers.test.ts => sequencers.test.ts} | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename test/core/test/{sequelizers.test.ts => sequencers.test.ts} (87%) diff --git a/test/core/test/sequelizers.test.ts b/test/core/test/sequencers.test.ts similarity index 87% rename from test/core/test/sequelizers.test.ts rename to test/core/test/sequencers.test.ts index 1980f266a709..5b1b84649ea9 100644 --- a/test/core/test/sequelizers.test.ts +++ b/test/core/test/sequencers.test.ts @@ -1,6 +1,6 @@ import type { Vitest } from 'vitest' import { describe, expect, test, vi } from 'vitest' -import { BaseSequelizer } from '../../../packages/vitest/src/node/sequelizers/BaseSequelizer' +import { BaseSequencer } from '../../../packages/vitest/src/node/sequencers/BaseSequencer' const buildCtx = () => { return { @@ -13,7 +13,7 @@ const buildCtx = () => { describe('test sequelizers', () => { test('sorting when no info is available', async () => { - const sequelizer = new BaseSequelizer(buildCtx()) + const sequelizer = new BaseSequencer(buildCtx()) const files = ['a', 'b', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(files) @@ -25,7 +25,7 @@ describe('test sequelizers', () => { if (file === 'b') return { size: 2 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequelizer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(['a', 'c', 'b']) @@ -41,7 +41,7 @@ describe('test sequelizers', () => { if (file === 'c') return { size: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequelizer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) @@ -57,7 +57,7 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 1 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequelizer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(['b', 'c', 'a']) @@ -73,7 +73,7 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequelizer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) @@ -89,7 +89,7 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequelizer(ctx) + const sequelizer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] const sorted = await sequelizer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) From 827e106fa9ec7523dd3a45a2c4b32e38d5791816 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:20:17 +0300 Subject: [PATCH 03/11] chore: add random option to suite --- docs/api/index.md | 17 ++++++++ docs/config/index.md | 31 +++++++++++++ docs/guide/cli.md | 5 +-- packages/vitest/src/node/cli.ts | 1 + packages/vitest/src/node/config.ts | 1 + packages/vitest/src/node/core.ts | 5 ++- .../src/node/sequencers/RandomSequencer.ts | 20 ++------- packages/vitest/src/runtime/run.ts | 12 +++++- packages/vitest/src/runtime/suite.ts | 17 +++++--- packages/vitest/src/types/config.ts | 2 +- packages/vitest/src/types/tasks.ts | 3 +- packages/vitest/src/utils/base.ts | 21 +++++++++ test/core/package.json | 2 +- test/core/test/random.test.ts | 31 +++++++++++++ test/core/test/sequencers.test.ts | 43 +++++++++++++------ test/core/vitest.config.ts | 3 ++ 16 files changed, 168 insertions(+), 46 deletions(-) create mode 100644 test/core/test/random.test.ts diff --git a/docs/api/index.md b/docs/api/index.md index 259b47820a42..9b84d42172ab 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -312,6 +312,23 @@ When you use `test` in the top level of file, they are collected as part of the describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */) ``` +### describe.random + +- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` + + Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.random`](/config/#sequence-random), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. + + ```ts + describe.random('suite', () => { + test('concurrent test 1', async () => { /* ... */ }) + test('concurrent test 2', async () => { /* ... */ }) + test('concurrent test 3', async () => { /* ... */ }) + }) + // order depends on sequence.seed option in config (Date.now() by default) + ``` + +`.skip`, `.only`, and `.todo` works with random suites. + ### describe.todo - **Type:** `(name: string) => void` diff --git a/docs/config/index.md b/docs/config/index.md index cb8b8eacc7ac..fbf90e33d3dc 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -582,3 +582,34 @@ Options to configure Vitest cache policy. At the moment Vitest stores cache for - **Default**: `node_modules/.vitest` Path to cache directory. + +### sequence + +- **Type**: `{ sequencer?, random?, seed? }` + +Options for how tests should be sorted. + +#### sequence.sequencer + +- **Type**: `TestSequencerConstructor` +- **Default**: `BaseSequencer` + +A custom class that defines methods for sharding and sorting. You can extend `BaseSequencer` from `vitest/node`, if you only need to redefine one of the `sort` and `shard` methods, but both should exist. + +Sharding is happening before sorting, and only if `--shard` option is provided. + +#### sequence.random + +- **Type**: `boolean` +- **Default**: `false` + +If you want tests to run randomly, you can enable it with this option, or CLI argument [`--random`](/guide/cli). + +Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. + +#### sequence.seed + +- **Type**: `number` +- **Default**: `Date.now()` + +Sets the randomization seed, if tests are running in random order. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index e43fccc19775..62a52d5665c0 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -36,10 +36,6 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit vitest related /src/index.ts /src/hello-world.js ``` -### `vitest clean cache` - -Clears cache folder. - ## Options | Options | | @@ -72,6 +68,7 @@ Clears cache folder. | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) | | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) | | `--shard ` | Execute tests in a specified shard | +| `--random` | Execute tests in random order | | `-h, --help` | Display available CLI options | ### changed diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 496f4ce88f28..d2dc1087383e 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -35,6 +35,7 @@ cli .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)') .option('--shard ', 'Test suite shard to execute in a format of /') .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') + .option('--random', 'Run tests in random order (default: false)') .help() cli diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 3df329b8f8fb..87087d31a056 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -191,6 +191,7 @@ export function resolveConfig( // random should have the priority over config, because it is a CLI flag if (!resolved.sequence?.sequencer || resolved.random) { resolved.sequence ??= {} as any + resolved.sequence.random ??= resolved.random resolved.sequence.sequencer = resolved.sequence.random || resolved.random ? RandomSequencer : BaseSequencer diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 8db21b680b5b..0409ca5ef7f6 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -105,7 +105,10 @@ export class Vitest { resolveSnapshotPath: undefined, }, onConsoleLog: undefined!, - sequence: undefined!, + sequence: { + ...this.config.sequence, + sequencer: undefined!, + }, }, this.configOverride || {} as any, ) as ResolvedConfig diff --git a/packages/vitest/src/node/sequencers/RandomSequencer.ts b/packages/vitest/src/node/sequencers/RandomSequencer.ts index b7c8a569e34a..87ac84852353 100644 --- a/packages/vitest/src/node/sequencers/RandomSequencer.ts +++ b/packages/vitest/src/node/sequencers/RandomSequencer.ts @@ -1,26 +1,12 @@ +import { randomize } from '../../utils' import { BaseSequencer } from './BaseSequencer' export class RandomSequencer extends BaseSequencer { - private random(seed: number) { - const x = Math.sin(seed++) * 10000 - return x - Math.floor(x) - } - public async sort(files: string[]) { const { sequence } = this.ctx.config - let seed = sequence?.seed ?? Date.now() - let length = files.length - - while (length) { - const index = Math.floor(this.random(seed) * length--) - - const previous = files[length] - files[length] = files[index] - files[index] = previous - ++seed - } + const seed = sequence?.seed ?? Date.now() - return files + return randomize(files, seed) } } diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index b3cd233110e6..bcc11ef9c65e 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -2,7 +2,7 @@ import limit from 'p-limit' import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' import { vi } from '../integrations/vi' import { getSnapshotClient } from '../integrations/snapshot/chai' -import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout } from '../utils' +import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, randomize, setTimeout } from '../utils' import { takeCoverage } from '../integrations/coverage' import { getState, setState } from '../integrations/chai/jest-expect' import { GLOBAL_EXPECT } from '../integrations/chai/constants' @@ -210,12 +210,20 @@ export async function runSuite(suite: Suite) { try { const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite]) - for (const tasksGroup of partitionSuiteChildren(suite)) { + for (let tasksGroup of partitionSuiteChildren(suite)) { if (tasksGroup[0].concurrent === true) { const mutex = limit(workerState.config.maxConcurrency) await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c)))) } else { + const { sequence } = workerState.config + if (sequence.random || suite.random) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = randomize([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => randomize(group, sequence.seed)) + } for (const c of tasksGroup) await runSuiteChild(c) } diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 99d8b394339e..35113aec0712 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -1,6 +1,6 @@ import { format } from 'util' import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types' -import { isObject, noop } from '../utils' +import { getWorkerState, isObject, noop } from '../utils' import { createChainable } from './chain' import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context' import { getHooks, setFn, setHooks } from './map' @@ -37,8 +37,12 @@ function formatTitle(template: string, items: any[], idx: number) { export const describe = suite export const it = test +const workerState = getWorkerState() + // implementations -export const defaultSuite = suite('') +export const defaultSuite = workerState.config.sequence.random + ? suite.random('') + : suite('') export function clearCollectorContext() { collectorContext.tasks.length = 0 @@ -59,7 +63,7 @@ export function createSuiteHooks() { } } -function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean) { +function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, random?: boolean) { const tasks: (Test | Suite | SuiteCollector)[] = [] const factoryQueue: (Test | Suite | SuiteCollector)[] = [] @@ -80,6 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } as Omit as Test if (this.concurrent || concurrent) test.concurrent = true + if (random) + test.random = true const context = createTestContext(test) // create test context @@ -117,6 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m type: 'suite', name, mode, + random, tasks: [], } setHooks(suite, createSuiteHooks()) @@ -157,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { const suite = createChainable( - ['concurrent', 'skip', 'only', 'todo'], + ['concurrent', 'random', 'skip', 'only', 'todo'], function (name: string, factory?: SuiteFactory) { const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' - return createSuiteCollector(name, factory, mode, this.concurrent) + return createSuiteCollector(name, factory, mode, this.concurrent, this.random) }, ) as SuiteAPI diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 5ebcf13849db..08370a8c9405 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -442,7 +442,7 @@ export interface UserConfig extends InlineConfig { * If tests should be run in random. * @default false */ - random?: string + random?: boolean } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index aaa91746bfc9..afc78ef83e38 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -10,6 +10,7 @@ export interface TaskBase { name: string mode: RunMode concurrent?: boolean + random?: boolean suite?: Suite file?: File result?: TaskResult @@ -113,7 +114,7 @@ void } export type SuiteAPI = ChainableFunction< -'concurrent' | 'only' | 'skip' | 'todo', +'concurrent' | 'only' | 'skip' | 'todo' | 'random', [name: string, factory?: SuiteFactory], SuiteCollector > & { diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index f872c3d74372..931e4f65769d 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -153,3 +153,24 @@ export function stdout(): NodeJS.WriteStream { // eslint-disable-next-line no-console return console._stdout || process.stdout } + +function random(seed: number) { + const x = Math.sin(seed++) * 10000 + return x - Math.floor(x) +} + +export function randomize(array: T[], seed?: number): T[] { + let length = array.length + seed ??= Date.now() + + while (length) { + const index = Math.floor(random(seed) * length--) + + const previous = array[length] + array[length] = array[index] + array[index] = previous + ++seed + } + + return array +} diff --git a/test/core/package.json b/test/core/package.json index 04d6e0b02701..aee453e7d942 100644 --- a/test/core/package.json +++ b/test/core/package.json @@ -2,7 +2,7 @@ "name": "@vitest/test-core", "private": true, "scripts": { - "test": "vitest", + "test": "vitest test/core/test/random.test.ts", "coverage": "vitest run --coverage" }, "devDependencies": { diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts new file mode 100644 index 000000000000..dc9284ba635f --- /dev/null +++ b/test/core/test/random.test.ts @@ -0,0 +1,31 @@ +import { afterAll, describe, expect, test } from 'vitest' + +// tests use seed of 101, so they have deterministic random order +const numbers: number[] = [] + +describe.random('random tests', () => { + describe('inside', () => { + // random is not inhereted from parent + + test('inside 1', () => { + numbers.push(1) + }) + test('inside 2', () => { + numbers.push(2) + }) + }) + + test('test 1', () => { + numbers.push(3) + }) + test('test 2', () => { + numbers.push(4) + }) + test('test 3', () => { + numbers.push(5) + }) + + afterAll(() => { + expect(numbers).toStrictEqual([4, 5, 3, 1, 2]) + }) +}) diff --git a/test/core/test/sequencers.test.ts b/test/core/test/sequencers.test.ts index 5b1b84649ea9..745ae8e7da03 100644 --- a/test/core/test/sequencers.test.ts +++ b/test/core/test/sequencers.test.ts @@ -1,9 +1,13 @@ import type { Vitest } from 'vitest' import { describe, expect, test, vi } from 'vitest' -import { BaseSequencer } from '../../../packages/vitest/src/node/sequencers/BaseSequencer' +import { RandomSequencer } from 'vitest/src/node/sequencers/RandomSequencer' +import { BaseSequencer } from 'vitest/src/node/sequencers/BaseSequencer' const buildCtx = () => { return { + config: { + sequence: {}, + }, state: { getFileTestResults: vi.fn(), getFileStats: vi.fn(), @@ -11,11 +15,11 @@ const buildCtx = () => { } as unknown as Vitest } -describe('test sequelizers', () => { +describe('base sequencer', () => { test('sorting when no info is available', async () => { - const sequelizer = new BaseSequencer(buildCtx()) + const sequencer = new BaseSequencer(buildCtx()) const files = ['a', 'b', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(files) }) @@ -25,9 +29,9 @@ describe('test sequelizers', () => { if (file === 'b') return { size: 2 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['a', 'c', 'b']) }) @@ -41,9 +45,9 @@ describe('test sequelizers', () => { if (file === 'c') return { size: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -57,9 +61,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 1 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['b', 'c', 'a']) }) @@ -73,9 +77,9 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) @@ -89,9 +93,20 @@ describe('test sequelizers', () => { if (file === 'c') return { failed: true, duration: 3 } }) - const sequelizer = new BaseSequencer(ctx) + const sequencer = new BaseSequencer(ctx) const files = ['b', 'a', 'c'] - const sorted = await sequelizer.sort(files) + const sorted = await sequencer.sort(files) expect(sorted).toStrictEqual(['c', 'b', 'a']) }) }) + +describe('random sequencer', () => { + test('sorting is the same when seed is defined', async () => { + const ctx = buildCtx() + ctx.config.sequence.seed = 101 + const sequencer = new RandomSequencer(ctx) + const files = ['b', 'a', 'c'] + const sorted = await sequencer.sort(files) + expect(sorted).toStrictEqual(['a', 'c', 'b']) + }) +}) diff --git a/test/core/vitest.config.ts b/test/core/vitest.config.ts index e6ca56714997..4b41690a8289 100644 --- a/test/core/vitest.config.ts +++ b/test/core/vitest.config.ts @@ -53,5 +53,8 @@ export default defineConfig({ return path + extension return join(dirname(path), '__snapshots__', `${basename(path)}${extension}`) }, + sequence: { + seed: 101, + }, }, }) From 941c943735db671e5af59faff7b5f89d1b22f973 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:23:27 +0300 Subject: [PATCH 04/11] chore: cleanup --- test/core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/package.json b/test/core/package.json index aee453e7d942..04d6e0b02701 100644 --- a/test/core/package.json +++ b/test/core/package.json @@ -2,7 +2,7 @@ "name": "@vitest/test-core", "private": true, "scripts": { - "test": "vitest test/core/test/random.test.ts", + "test": "vitest", "coverage": "vitest run --coverage" }, "devDependencies": { From a00e17a608d441351c1669fbeb48a57c4ecf1836 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:25:41 +0300 Subject: [PATCH 05/11] chore: cleanup --- docs/api/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 9b84d42172ab..c688a0566790 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -320,9 +320,9 @@ When you use `test` in the top level of file, they are collected as part of the ```ts describe.random('suite', () => { - test('concurrent test 1', async () => { /* ... */ }) - test('concurrent test 2', async () => { /* ... */ }) - test('concurrent test 3', async () => { /* ... */ }) + test('random test 1', async () => { /* ... */ }) + test('random test 2', async () => { /* ... */ }) + test('random test 3', async () => { /* ... */ }) }) // order depends on sequence.seed option in config (Date.now() by default) ``` From 5b0ac9de168586dc5607cae02b6a52d7746462db Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:26:32 +0300 Subject: [PATCH 06/11] chore: cleanup --- packages/vitest/src/utils/base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index 931e4f65769d..31ce1309bdde 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -1,3 +1,4 @@ +import { RealDate } from '../integrations/mock/date' import type { Arrayable, DeepMerge, Nullable } from '../types' function isFinalObj(obj: any) { @@ -161,7 +162,7 @@ function random(seed: number) { export function randomize(array: T[], seed?: number): T[] { let length = array.length - seed ??= Date.now() + seed ??= RealDate.now() while (length) { const index = Math.floor(random(seed) * length--) From 9ee2910f2262b0a6d59f23201034fc523abd11fc Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:50:16 +0300 Subject: [PATCH 07/11] chore: cleanup --- packages/vitest/src/utils/base.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index 31ce1309bdde..d70354605a2b 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -160,9 +160,8 @@ function random(seed: number) { return x - Math.floor(x) } -export function randomize(array: T[], seed?: number): T[] { +export function randomize(array: T[], seed = RealDate.now()): T[] { let length = array.length - seed ??= RealDate.now() while (length) { const index = Math.floor(random(seed) * length--) From 43a5f9a673f49532211e3e31971f9533cfdd506d Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 4 Jul 2022 21:52:13 +0300 Subject: [PATCH 08/11] chore: cleanup --- packages/vitest/src/node/config.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 87087d31a056..32642b96ce68 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -191,8 +191,10 @@ export function resolveConfig( // random should have the priority over config, because it is a CLI flag if (!resolved.sequence?.sequencer || resolved.random) { resolved.sequence ??= {} as any - resolved.sequence.random ??= resolved.random - resolved.sequence.sequencer = resolved.sequence.random || resolved.random + // CLI flag has higher priority + if (resolved.random) + resolved.sequence.random = true + resolved.sequence.sequencer = resolved.sequence.random ? RandomSequencer : BaseSequencer } From 3715ab467027b079f7b9081d53b8814e4e826350 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Wed, 6 Jul 2022 09:52:51 +0300 Subject: [PATCH 09/11] chore: rename random -> shuffle --- docs/api/index.md | 6 +++--- docs/config/index.md | 6 +++--- docs/guide/cli.md | 2 +- packages/vitest/src/node/cli.ts | 2 +- packages/vitest/src/node/config.ts | 7 ++----- .../src/node/sequencers/RandomSequencer.ts | 4 ++-- packages/vitest/src/runtime/run.ts | 8 ++++---- packages/vitest/src/runtime/suite.ts | 16 ++++++++-------- packages/vitest/src/types/config.ts | 10 ++-------- packages/vitest/src/types/tasks.ts | 4 ++-- packages/vitest/src/utils/base.ts | 2 +- test/core/test/random.test.ts | 4 ++-- 12 files changed, 31 insertions(+), 40 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index c688a0566790..f80bc3405ff5 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -312,14 +312,14 @@ When you use `test` in the top level of file, they are collected as part of the describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */) ``` -### describe.random +### describe.shuffle - **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` - Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.random`](/config/#sequence-random), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. + Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. ```ts - describe.random('suite', () => { + describe.shuffle('suite', () => { test('random test 1', async () => { /* ... */ }) test('random test 2', async () => { /* ... */ }) test('random test 3', async () => { /* ... */ }) diff --git a/docs/config/index.md b/docs/config/index.md index fbf90e33d3dc..31cadf6c7ccc 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -585,7 +585,7 @@ Path to cache directory. ### sequence -- **Type**: `{ sequencer?, random?, seed? }` +- **Type**: `{ sequencer?, shuffle?, seed? }` Options for how tests should be sorted. @@ -598,12 +598,12 @@ A custom class that defines methods for sharding and sorting. You can extend `Ba Sharding is happening before sorting, and only if `--shard` option is provided. -#### sequence.random +#### sequence.shuffle - **Type**: `boolean` - **Default**: `false` -If you want tests to run randomly, you can enable it with this option, or CLI argument [`--random`](/guide/cli). +If you want tests to run randomly, you can enable it with this option, or CLI argument [`--shuffle`](/guide/cli). Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 62a52d5665c0..614d06ad572a 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -68,7 +68,7 @@ vitest related /src/index.ts /src/hello-world.js | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) | | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) | | `--shard ` | Execute tests in a specified shard | -| `--random` | Execute tests in random order | +| `--sequence` | Define in what order to run tests. Use [cac's dot notation] to specify options (for example, use `--sequence.suffle` to run tests in random order) | | `-h, --help` | Display available CLI options | ### changed diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index d2dc1087383e..a8002f2373f1 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -35,7 +35,7 @@ cli .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)') .option('--shard ', 'Test suite shard to execute in a format of /') .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') - .option('--random', 'Run tests in random order (default: false)') + .option('--sequence ', 'Define in what order to run tests (use --sequence.shuffle to run tests in random order)') .help() cli diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 32642b96ce68..ec3212e0fbf2 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -188,13 +188,10 @@ export function resolveConfig( if (resolved.cache) resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir) - // random should have the priority over config, because it is a CLI flag - if (!resolved.sequence?.sequencer || resolved.random) { + if (!resolved.sequence?.sequencer) { resolved.sequence ??= {} as any // CLI flag has higher priority - if (resolved.random) - resolved.sequence.random = true - resolved.sequence.sequencer = resolved.sequence.random + resolved.sequence.sequencer = resolved.sequence.shuffle ? RandomSequencer : BaseSequencer } diff --git a/packages/vitest/src/node/sequencers/RandomSequencer.ts b/packages/vitest/src/node/sequencers/RandomSequencer.ts index 87ac84852353..c92c82f62080 100644 --- a/packages/vitest/src/node/sequencers/RandomSequencer.ts +++ b/packages/vitest/src/node/sequencers/RandomSequencer.ts @@ -1,4 +1,4 @@ -import { randomize } from '../../utils' +import { shuffle } from '../../utils' import { BaseSequencer } from './BaseSequencer' export class RandomSequencer extends BaseSequencer { @@ -7,6 +7,6 @@ export class RandomSequencer extends BaseSequencer { const seed = sequence?.seed ?? Date.now() - return randomize(files, seed) + return shuffle(files, seed) } } diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index bcc11ef9c65e..d6bc51fc2830 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -2,7 +2,7 @@ import limit from 'p-limit' import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' import { vi } from '../integrations/vi' import { getSnapshotClient } from '../integrations/snapshot/chai' -import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, randomize, setTimeout } from '../utils' +import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout, shuffle } from '../utils' import { takeCoverage } from '../integrations/coverage' import { getState, setState } from '../integrations/chai/jest-expect' import { GLOBAL_EXPECT } from '../integrations/chai/constants' @@ -217,12 +217,12 @@ export async function runSuite(suite: Suite) { } else { const { sequence } = workerState.config - if (sequence.random || suite.random) { + if (sequence.shuffle || suite.shuffle) { // run describe block independently from tests const suites = tasksGroup.filter(group => group.type === 'suite') const tests = tasksGroup.filter(group => group.type === 'test') - const groups = randomize([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => randomize(group, sequence.seed)) + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) } for (const c of tasksGroup) await runSuiteChild(c) diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 35113aec0712..e3f464f20085 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -40,8 +40,8 @@ export const it = test const workerState = getWorkerState() // implementations -export const defaultSuite = workerState.config.sequence.random - ? suite.random('') +export const defaultSuite = workerState.config.sequence.shuffle + ? suite.shuffle('') : suite('') export function clearCollectorContext() { @@ -63,7 +63,7 @@ export function createSuiteHooks() { } } -function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, random?: boolean) { +function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, shuffle?: boolean) { const tasks: (Test | Suite | SuiteCollector)[] = [] const factoryQueue: (Test | Suite | SuiteCollector)[] = [] @@ -84,8 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m } as Omit as Test if (this.concurrent || concurrent) test.concurrent = true - if (random) - test.random = true + if (shuffle) + test.shuffle = true const context = createTestContext(test) // create test context @@ -123,7 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m type: 'suite', name, mode, - random, + shuffle, tasks: [], } setHooks(suite, createSuiteHooks()) @@ -164,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m function createSuite() { const suite = createChainable( - ['concurrent', 'random', 'skip', 'only', 'todo'], + ['concurrent', 'shuffle', 'skip', 'only', 'todo'], function (name: string, factory?: SuiteFactory) { const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' - return createSuiteCollector(name, factory, mode, this.concurrent, this.random) + return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle) }, ) as SuiteAPI diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 08370a8c9405..9f0bb9ebd44e 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -386,7 +386,7 @@ export interface InlineConfig { * Should tests run in random order. * @default false */ - random?: boolean + shuffle?: boolean /** * Seed for the random number generator. * @default Date.now() @@ -437,12 +437,6 @@ export interface UserConfig extends InlineConfig { * @example --shard=2/3 */ shard?: string - - /** - * If tests should be run in random. - * @default false - */ - random?: boolean } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { @@ -472,7 +466,7 @@ export interface ResolvedConfig extends Omit, 'config' | 'f sequence: { sequencer: TestSequencerContructor - random?: boolean + shuffle?: boolean seed?: number } } diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index afc78ef83e38..773601590d78 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -10,7 +10,7 @@ export interface TaskBase { name: string mode: RunMode concurrent?: boolean - random?: boolean + shuffle?: boolean suite?: Suite file?: File result?: TaskResult @@ -114,7 +114,7 @@ void } export type SuiteAPI = ChainableFunction< -'concurrent' | 'only' | 'skip' | 'todo' | 'random', +'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle', [name: string, factory?: SuiteFactory], SuiteCollector > & { diff --git a/packages/vitest/src/utils/base.ts b/packages/vitest/src/utils/base.ts index d70354605a2b..9633c670cfe0 100644 --- a/packages/vitest/src/utils/base.ts +++ b/packages/vitest/src/utils/base.ts @@ -160,7 +160,7 @@ function random(seed: number) { return x - Math.floor(x) } -export function randomize(array: T[], seed = RealDate.now()): T[] { +export function shuffle(array: T[], seed = RealDate.now()): T[] { let length = array.length while (length) { diff --git a/test/core/test/random.test.ts b/test/core/test/random.test.ts index dc9284ba635f..fefc9594b812 100644 --- a/test/core/test/random.test.ts +++ b/test/core/test/random.test.ts @@ -3,9 +3,9 @@ import { afterAll, describe, expect, test } from 'vitest' // tests use seed of 101, so they have deterministic random order const numbers: number[] = [] -describe.random('random tests', () => { +describe.shuffle('random tests', () => { describe('inside', () => { - // random is not inhereted from parent + // shuffle is not inhereted from parent test('inside 1', () => { numbers.push(1) From 900175c430efb59e29afa10e7dbb557d600ad65a Mon Sep 17 00:00:00 2001 From: Vladimir Date: Wed, 6 Jul 2022 12:57:32 +0300 Subject: [PATCH 10/11] Apply suggestions from code review Co-authored-by: Anthony Fu --- docs/api/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/index.md b/docs/api/index.md index f80bc3405ff5..64fb061caca6 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -316,7 +316,7 @@ When you use `test` in the top level of file, they are collected as part of the - **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` - Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. + Vitest provides a way to run all tests in random order via CLI flag [`--sequence.shuffle`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. ```ts describe.shuffle('suite', () => { From 8cade607cbe148a8bc4e7c7d2c5eb09a00e3ff10 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Wed, 6 Jul 2022 12:57:45 +0300 Subject: [PATCH 11/11] Update docs/config/index.md Co-authored-by: Anthony Fu --- docs/config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config/index.md b/docs/config/index.md index 31cadf6c7ccc..a2850705c67f 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -603,7 +603,7 @@ Sharding is happening before sorting, and only if `--shard` option is provided. - **Type**: `boolean` - **Default**: `false` -If you want tests to run randomly, you can enable it with this option, or CLI argument [`--shuffle`](/guide/cli). +If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.