feat: make sequencer public, add option to run tests in random order

docs/api/index.md

@@ -312,6 +312,23 @@ When you use `test` in the top level of file, they are collected as part of the
   describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */)
   ```
 
+### describe.shuffle
+
+- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void`
+
+  Vitest provides a way to run all tests in random order via CLI flag [`--sequence.shuffle`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag.
+
+  ```ts
+  describe.shuffle('suite', () => {
+    test('random test 1', async () => { /* ... */ })
+    test('random test 2', async () => { /* ... */ })
+    test('random test 3', async () => { /* ... */ })
+  })
+  // order depends on sequence.seed option in config (Date.now() by default)
+  ```
+
+`.skip`, `.only`, and `.todo` works with random suites.
+
 ### describe.todo
 
 - **Type:** `(name: string) => void`

docs/config/index.md

@@ -582,3 +582,34 @@ Options to configure Vitest cache policy. At the moment Vitest stores cache for
 - **Default**: `node_modules/.vitest`
 
 Path to cache directory.
+
+### sequence
+
+- **Type**: `{ sequencer?, shuffle?, seed? }`
+
+Options for how tests should be sorted.
+
+#### sequence.sequencer
+
+- **Type**: `TestSequencerConstructor`
+- **Default**: `BaseSequencer`
+
+A custom class that defines methods for sharding and sorting. You can extend `BaseSequencer` from `vitest/node`, if you only need to redefine one of the `sort` and `shard` methods, but both should exist.
+
+Sharding is happening before sorting, and only if `--shard` option is provided.
+
+#### sequence.shuffle
+
+- **Type**: `boolean`
+- **Default**: `false`
+
+If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli).
+
+Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
+
+#### sequence.seed
+
+- **Type**: `number`
+- **Default**: `Date.now()`
+
+Sets the randomization seed, if tests are running in random order.

docs/guide/cli.md

@@ -36,10 +36,6 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit
 vitest related /src/index.ts /src/hello-world.js
 ```
 
-### `vitest clean cache`
-
-Clears cache folder.
-
 ## Options
 
 | Options       |               |
@@ -72,6 +68,7 @@ Clears cache folder.
 | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) |
 | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) |
 | `--shard <shard>` | Execute tests in a specified shard |
+| `--sequence` | Define in what order to run tests. Use [cac's dot notation] to specify options (for example, use `--sequence.suffle` to run tests in random order) |
 | `-h, --help` | Display available CLI options |
 
 ### changed

packages/vitest/src/node/cli.ts

@@ -35,6 +35,7 @@ cli
   .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)')
   .option('--shard <shard>', 'Test suite shard to execute in a format of <index>/<count>')
   .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)')
+  .option('--sequence <options>', 'Define in what order to run tests (use --sequence.shuffle to run tests in random order)')
   .help()
 
 cli

packages/vitest/src/node/config.ts

@@ -9,6 +9,8 @@ import { configDefaults } from '../defaults'
 import { resolveC8Options } from '../integrations/coverage'
 import { toArray } from '../utils'
 import { VitestCache } from './cache'
+import { BaseSequencer } from './sequencers/BaseSequencer'
+import { RandomSequencer } from './sequencers/RandomSequencer'
 
 const extraInlineDeps = [
   /^(?!.*(?:node_modules)).*\.mjs$/,
@@ -186,5 +188,13 @@ export function resolveConfig(
   if (resolved.cache)
     resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir)
 
+  if (!resolved.sequence?.sequencer) {
+    resolved.sequence ??= {} as any
+    // CLI flag has higher priority
+    resolved.sequence.sequencer = resolved.sequence.shuffle
+      ? RandomSequencer
+      : BaseSequencer
+  }
+
   return resolved
 }

packages/vitest/src/node/core.ts

@@ -105,6 +105,10 @@ export class Vitest {
         resolveSnapshotPath: undefined,
       },
       onConsoleLog: undefined!,
+      sequence: {
+        ...this.config.sequence,
+        sequencer: undefined!,
+      },
     },
     this.configOverride || {} as any,
     ) as ResolvedConfig

packages/vitest/src/node/index.ts

@@ -5,3 +5,6 @@ export { startVitest } from './cli-api'
 
 export { VitestRunner } from '../runtime/execute'
 export type { ExecuteOptions } from '../runtime/execute'
+
+export type { TestSequencer, TestSequencerContructor } from './sequencers/types'
+export { BaseSequencer } from './sequencers/BaseSequencer'

packages/vitest/src/node/pool.ts

@@ -10,7 +10,6 @@ import type { ResolvedConfig, WorkerContext, WorkerRPC } from '../types'
 import { distDir } from '../constants'
 import { AggregateError } from '../utils'
 import type { Vitest } from './core'
-import { BaseSequelizer } from './sequelizers/BaseSequelizer'
 
 export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise<void>
 
@@ -86,15 +85,16 @@ export function createPool(ctx: Vitest): WorkerPool {
       }
     }
 
-    const sequelizer = new BaseSequelizer(ctx)
+    const Sequencer = ctx.config.sequence.sequencer
+    const sequencer = new Sequencer(ctx)
 
     return async (files, invalidates) => {
       const config = ctx.getSerializableConfig()
 
       if (config.shard)
-        files = await sequelizer.shard(files)
+        files = await sequencer.shard(files)
 
-      files = await sequelizer.sort(files)
+      files = await sequencer.sort(files)
 
       if (!ctx.config.threads) {
         await runFiles(config, files)

packages/vitest/src/node/sequelizers/BaseSequelizer.ts → packages/vitest/src/node/sequencers/BaseSequencer.ts

@@ -2,9 +2,9 @@ import { createHash } from 'crypto'
 import { resolve } from 'pathe'
 import { slash } from 'vite-node/utils'
 import type { Vitest } from '../core'
-import type { TestSequelizer } from './types'
+import type { TestSequencer } from './types'
 
-export class BaseSequelizer implements TestSequelizer {
+export class BaseSequencer implements TestSequencer {
   protected ctx: Vitest
 
   constructor(ctx: Vitest) {

packages/vitest/src/node/sequencers/RandomSequencer.ts

@@ -0,0 +1,12 @@
+import { shuffle } from '../../utils'
+import { BaseSequencer } from './BaseSequencer'
+
+export class RandomSequencer extends BaseSequencer {
+  public async sort(files: string[]) {
+    const { sequence } = this.ctx.config
+
+    const seed = sequence?.seed ?? Date.now()
+
+    return shuffle(files, seed)
+  }
+}

packages/vitest/src/node/sequelizers/types.ts → packages/vitest/src/node/sequencers/types.ts

@@ -1,7 +1,7 @@
 import type { Awaitable } from '../../types'
 import type { Vitest } from '../core'
 
-export interface TestSequelizer {
+export interface TestSequencer {
   /**
    * Slicing tests into shards. Will be run before `sort`.
    * Only run, if `shard` is defined.
@@ -10,6 +10,6 @@ export interface TestSequelizer {
   sort(files: string[]): Awaitable<string[]>
 }
 
-export interface TestSequelizerContructor {
-  new (ctx: Vitest): TestSequelizer
+export interface TestSequencerContructor {
+  new (ctx: Vitest): TestSequencer
 }

packages/vitest/src/runtime/run.ts

@@ -2,7 +2,7 @@ import limit from 'p-limit'
 import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types'
 import { vi } from '../integrations/vi'
 import { getSnapshotClient } from '../integrations/snapshot/chai'
-import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout } from '../utils'
+import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout, shuffle } from '../utils'
 import { takeCoverage } from '../integrations/coverage'
 import { getState, setState } from '../integrations/chai/jest-expect'
 import { GLOBAL_EXPECT } from '../integrations/chai/constants'
@@ -210,12 +210,20 @@ export async function runSuite(suite: Suite) {
     try {
       const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite])
 
-      for (const tasksGroup of partitionSuiteChildren(suite)) {
+      for (let tasksGroup of partitionSuiteChildren(suite)) {
         if (tasksGroup[0].concurrent === true) {
           const mutex = limit(workerState.config.maxConcurrency)
           await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c))))
         }
         else {
+          const { sequence } = workerState.config
+          if (sequence.shuffle || suite.shuffle) {
+            // run describe block independently from tests
+            const suites = tasksGroup.filter(group => group.type === 'suite')
+            const tests = tasksGroup.filter(group => group.type === 'test')
+            const groups = shuffle([suites, tests], sequence.seed)
+            tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed))
+          }
           for (const c of tasksGroup)
             await runSuiteChild(c)
         }

packages/vitest/src/runtime/suite.ts

@@ -1,6 +1,6 @@
 import { format } from 'util'
 import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types'
-import { isObject, noop } from '../utils'
+import { getWorkerState, isObject, noop } from '../utils'
 import { createChainable } from './chain'
 import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context'
 import { getHooks, setFn, setHooks } from './map'
@@ -37,8 +37,12 @@ function formatTitle(template: string, items: any[], idx: number) {
 export const describe = suite
 export const it = test
 
+const workerState = getWorkerState()
+
 // implementations
-export const defaultSuite = suite('')
+export const defaultSuite = workerState.config.sequence.shuffle
+  ? suite.shuffle('')
+  : suite('')
 
 export function clearCollectorContext() {
   collectorContext.tasks.length = 0
@@ -59,7 +63,7 @@ export function createSuiteHooks() {
   }
 }
 
-function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean) {
+function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, shuffle?: boolean) {
   const tasks: (Test | Suite | SuiteCollector)[] = []
   const factoryQueue: (Test | Suite | SuiteCollector)[] = []
 
@@ -80,6 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
     } as Omit<Test, 'context'> as Test
     if (this.concurrent || concurrent)
       test.concurrent = true
+    if (shuffle)
+      test.shuffle = true
 
     const context = createTestContext(test)
     // create test context
@@ -117,6 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       type: 'suite',
       name,
       mode,
+      shuffle,
       tasks: [],
     }
     setHooks(suite, createSuiteHooks())
@@ -157,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
 
 function createSuite() {
   const suite = createChainable(
-    ['concurrent', 'skip', 'only', 'todo'],
+    ['concurrent', 'shuffle', 'skip', 'only', 'todo'],
     function (name: string, factory?: SuiteFactory) {
       const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run'
-      return createSuiteCollector(name, factory, mode, this.concurrent)
+      return createSuiteCollector(name, factory, mode, this.concurrent, this.shuffle)
     },
   ) as SuiteAPI
 

packages/vitest/src/types/config.ts

@@ -2,6 +2,7 @@ import type { CommonServerOptions } from 'vite'
 import type { PrettyFormatOptions } from 'pretty-format'
 import type { FakeTimerInstallOpts } from '@sinonjs/fake-timers'
 import type { BuiltinReporters } from '../node/reporters'
+import type { TestSequencerContructor } from '../node/sequencers/types'
 import type { C8Options, ResolvedC8Options } from './coverage'
 import type { JSDOMOptions } from './jsdom-options'
 import type { Reporter } from './reporter'
@@ -369,6 +370,29 @@ export interface InlineConfig {
   cache?: false | {
     dir?: string
   }
+
+  /**
+   * Options for configuring the order of running tests.
+   */
+  sequence?: {
+    /**
+     * Class that handles sorting and sharding algorithm.
+     * If you only need to change sorting, you can extend
+     * your custom sequencer from `BaseSequencer` from `vitest/node`.
+     * @default BaseSequencer
+     */
+    sequencer?: TestSequencerContructor
+    /**
+     * Should tests run in random order.
+     * @default false
+     */
+    shuffle?: boolean
+    /**
+     * Seed for the random number generator.
+     * @default Date.now()
+     */
+    seed?: number
+  }
 }
 
 export interface UserConfig extends InlineConfig {
@@ -415,7 +439,7 @@ export interface UserConfig extends InlineConfig {
   shard?: string
 }
 
-export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache'> {
+export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> {
   base?: string
 
   config?: string
@@ -439,4 +463,10 @@ export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'f
   cache: {
     dir: string
   } | false
+
+  sequence: {
+    sequencer: TestSequencerContructor
+    shuffle?: boolean
+    seed?: number
+  }
 }

packages/vitest/src/types/tasks.ts

@@ -10,6 +10,7 @@ export interface TaskBase {
   name: string
   mode: RunMode
   concurrent?: boolean
+  shuffle?: boolean
   suite?: Suite
   file?: File
   result?: TaskResult
@@ -113,7 +114,7 @@ void
 }
 
 export type SuiteAPI<ExtraContext = {}> = ChainableFunction<
-'concurrent' | 'only' | 'skip' | 'todo',
+'concurrent' | 'only' | 'skip' | 'todo' | 'shuffle',
 [name: string, factory?: SuiteFactory],
 SuiteCollector<ExtraContext>
 > & {

packages/vitest/src/utils/base.ts

@@ -1,3 +1,4 @@
+import { RealDate } from '../integrations/mock/date'
 import type { Arrayable, DeepMerge, Nullable } from '../types'
 
 function isFinalObj(obj: any) {
@@ -153,3 +154,23 @@ export function stdout(): NodeJS.WriteStream {
   // eslint-disable-next-line no-console
   return console._stdout || process.stdout
 }
+
+function random(seed: number) {
+  const x = Math.sin(seed++) * 10000
+  return x - Math.floor(x)
+}
+
+export function shuffle<T>(array: T[], seed = RealDate.now()): T[] {
+  let length = array.length
+
+  while (length) {
+    const index = Math.floor(random(seed) * length--)
+
+    const previous = array[length]
+    array[length] = array[index]
+    array[index] = previous
+    ++seed
+  }
+
+  return array
+}