chore: add random option to suite

docs/api/index.md

@@ -312,6 +312,23 @@ When you use `test` in the top level of file, they are collected as part of the
   describe.todo.concurrent(/* ... */) // or describe.concurrent.todo(/* ... */)
   ```
 
+### describe.random
+
+- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void`
+
+  Vitest provides a way to run all tests in random order via CLI flag [`--random`](/guide/cli) or config option [`sequence.random`](/config/#sequence-random), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag.
+
+  ```ts
+  describe.random('suite', () => {
+    test('concurrent test 1', async () => { /* ... */ })
+    test('concurrent test 2', async () => { /* ... */ })
+    test('concurrent test 3', async () => { /* ... */ })
+  })
+  // order depends on sequence.seed option in config (Date.now() by default)
+  ```
+
+`.skip`, `.only`, and `.todo` works with random suites.
+
 ### describe.todo
 
 - **Type:** `(name: string) => void`

docs/config/index.md

@@ -582,3 +582,34 @@ Options to configure Vitest cache policy. At the moment Vitest stores cache for
 - **Default**: `node_modules/.vitest`
 
 Path to cache directory.
+
+### sequence
+
+- **Type**: `{ sequencer?, random?, seed? }`
+
+Options for how tests should be sorted.
+
+#### sequence.sequencer
+
+- **Type**: `TestSequencerConstructor`
+- **Default**: `BaseSequencer`
+
+A custom class that defines methods for sharding and sorting. You can extend `BaseSequencer` from `vitest/node`, if you only need to redefine one of the `sort` and `shard` methods, but both should exist.
+
+Sharding is happening before sorting, and only if `--shard` option is provided.
+
+#### sequence.random
+
+- **Type**: `boolean`
+- **Default**: `false`
+
+If you want tests to run randomly, you can enable it with this option, or CLI argument [`--random`](/guide/cli).
+
+Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
+
+#### sequence.seed
+
+- **Type**: `number`
+- **Default**: `Date.now()`
+
+Sets the randomization seed, if tests are running in random order.

docs/guide/cli.md

@@ -36,10 +36,6 @@ Useful to run with [`lint-staged`](https://github.com/okonet/lint-staged) or wit
 vitest related /src/index.ts /src/hello-world.js
 ```
 
-### `vitest clean cache`
-
-Clears cache folder.
-
 ## Options
 
 | Options       |               |
@@ -72,6 +68,7 @@ Clears cache folder.
 | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) |
 | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) |
 | `--shard <shard>` | Execute tests in a specified shard |
+| `--random` | Execute tests in random order |
 | `-h, --help` | Display available CLI options |
 
 ### changed

packages/vitest/src/node/cli.ts

@@ -35,6 +35,7 @@ cli
   .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)')
   .option('--shard <shard>', 'Test suite shard to execute in a format of <index>/<count>')
   .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)')
+  .option('--random', 'Run tests in random order (default: false)')
   .help()
 
 cli

packages/vitest/src/node/config.ts

@@ -191,6 +191,7 @@ export function resolveConfig(
   // random should have the priority over config, because it is a CLI flag
   if (!resolved.sequence?.sequencer || resolved.random) {
     resolved.sequence ??= {} as any
+    resolved.sequence.random ??= resolved.random
     resolved.sequence.sequencer = resolved.sequence.random || resolved.random
       ? RandomSequencer
       : BaseSequencer

packages/vitest/src/node/core.ts

@@ -105,7 +105,10 @@ export class Vitest {
         resolveSnapshotPath: undefined,
       },
       onConsoleLog: undefined!,
-      sequence: undefined!,
+      sequence: {
+        ...this.config.sequence,
+        sequencer: undefined!,
+      },
     },
     this.configOverride || {} as any,
     ) as ResolvedConfig

packages/vitest/src/node/sequencers/RandomSequencer.ts

@@ -1,26 +1,12 @@
+import { randomize } from '../../utils'
 import { BaseSequencer } from './BaseSequencer'
 
 export class RandomSequencer extends BaseSequencer {
-  private random(seed: number) {
-    const x = Math.sin(seed++) * 10000
-    return x - Math.floor(x)
-  }
-
   public async sort(files: string[]) {
     const { sequence } = this.ctx.config
 
-    let seed = sequence?.seed ?? Date.now()
-    let length = files.length
-
-    while (length) {
-      const index = Math.floor(this.random(seed) * length--)
-
-      const previous = files[length]
-      files[length] = files[index]
-      files[index] = previous
-      ++seed
-    }
+    const seed = sequence?.seed ?? Date.now()
 
-    return files
+    return randomize(files, seed)
   }
 }

packages/vitest/src/runtime/run.ts

@@ -2,7 +2,7 @@ import limit from 'p-limit'
 import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types'
 import { vi } from '../integrations/vi'
 import { getSnapshotClient } from '../integrations/snapshot/chai'
-import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, setTimeout } from '../utils'
+import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, partitionSuiteChildren, randomize, setTimeout } from '../utils'
 import { takeCoverage } from '../integrations/coverage'
 import { getState, setState } from '../integrations/chai/jest-expect'
 import { GLOBAL_EXPECT } from '../integrations/chai/constants'
@@ -210,12 +210,20 @@ export async function runSuite(suite: Suite) {
     try {
       const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite])
 
-      for (const tasksGroup of partitionSuiteChildren(suite)) {
+      for (let tasksGroup of partitionSuiteChildren(suite)) {
         if (tasksGroup[0].concurrent === true) {
           const mutex = limit(workerState.config.maxConcurrency)
           await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c))))
         }
         else {
+          const { sequence } = workerState.config
+          if (sequence.random || suite.random) {
+            // run describe block independently from tests
+            const suites = tasksGroup.filter(group => group.type === 'suite')
+            const tests = tasksGroup.filter(group => group.type === 'test')
+            const groups = randomize([suites, tests], sequence.seed)
+            tasksGroup = groups.flatMap(group => randomize(group, sequence.seed))
+          }
           for (const c of tasksGroup)
             await runSuiteChild(c)
         }

packages/vitest/src/runtime/suite.ts

@@ -1,6 +1,6 @@
 import { format } from 'util'
 import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types'
-import { isObject, noop } from '../utils'
+import { getWorkerState, isObject, noop } from '../utils'
 import { createChainable } from './chain'
 import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context'
 import { getHooks, setFn, setHooks } from './map'
@@ -37,8 +37,12 @@ function formatTitle(template: string, items: any[], idx: number) {
 export const describe = suite
 export const it = test
 
+const workerState = getWorkerState()
+
 // implementations
-export const defaultSuite = suite('')
+export const defaultSuite = workerState.config.sequence.random
+  ? suite.random('')
+  : suite('')
 
 export function clearCollectorContext() {
   collectorContext.tasks.length = 0
@@ -59,7 +63,7 @@ export function createSuiteHooks() {
   }
 }
 
-function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean) {
+function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, random?: boolean) {
   const tasks: (Test | Suite | SuiteCollector)[] = []
   const factoryQueue: (Test | Suite | SuiteCollector)[] = []
 
@@ -80,6 +84,8 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
     } as Omit<Test, 'context'> as Test
     if (this.concurrent || concurrent)
       test.concurrent = true
+    if (random)
+      test.random = true
 
     const context = createTestContext(test)
     // create test context
@@ -117,6 +123,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       type: 'suite',
       name,
       mode,
+      random,
       tasks: [],
     }
     setHooks(suite, createSuiteHooks())
@@ -157,10 +164,10 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
 
 function createSuite() {
   const suite = createChainable(
-    ['concurrent', 'skip', 'only', 'todo'],
+    ['concurrent', 'random', 'skip', 'only', 'todo'],
     function (name: string, factory?: SuiteFactory) {
       const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run'
-      return createSuiteCollector(name, factory, mode, this.concurrent)
+      return createSuiteCollector(name, factory, mode, this.concurrent, this.random)
     },
   ) as SuiteAPI
 

packages/vitest/src/types/config.ts

@@ -442,7 +442,7 @@ export interface UserConfig extends InlineConfig {
    * If tests should be run in random.
    * @default false
    */
-  random?: string
+  random?: boolean
 }
 
 export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> {

packages/vitest/src/types/tasks.ts

@@ -10,6 +10,7 @@ export interface TaskBase {
   name: string
   mode: RunMode
   concurrent?: boolean
+  random?: boolean
   suite?: Suite
   file?: File
   result?: TaskResult
@@ -113,7 +114,7 @@ void
 }
 
 export type SuiteAPI<ExtraContext = {}> = ChainableFunction<
-'concurrent' | 'only' | 'skip' | 'todo',
+'concurrent' | 'only' | 'skip' | 'todo' | 'random',
 [name: string, factory?: SuiteFactory],
 SuiteCollector<ExtraContext>
 > & {

packages/vitest/src/utils/base.ts

@@ -153,3 +153,24 @@ export function stdout(): NodeJS.WriteStream {
   // eslint-disable-next-line no-console
   return console._stdout || process.stdout
 }
+
+function random(seed: number) {
+  const x = Math.sin(seed++) * 10000
+  return x - Math.floor(x)
+}
+
+export function randomize<T>(array: T[], seed?: number): T[] {
+  let length = array.length
+  seed ??= Date.now()
+
+  while (length) {
+    const index = Math.floor(random(seed) * length--)
+
+    const previous = array[length]
+    array[length] = array[index]
+    array[index] = previous
+    ++seed
+  }
+
+  return array
+}

test/core/package.json

@@ -2,7 +2,7 @@
   "name": "@vitest/test-core",
   "private": true,
   "scripts": {
-    "test": "vitest",
+    "test": "vitest test/core/test/random.test.ts",
     "coverage": "vitest run --coverage"
   },
   "devDependencies": {

test/core/test/random.test.ts

@@ -0,0 +1,31 @@
+import { afterAll, describe, expect, test } from 'vitest'
+
+// tests use seed of 101, so they have deterministic random order
+const numbers: number[] = []
+
+describe.random('random tests', () => {
+  describe('inside', () => {
+    // random is not inhereted from parent
+
+    test('inside 1', () => {
+      numbers.push(1)
+    })
+    test('inside 2', () => {
+      numbers.push(2)
+    })
+  })
+
+  test('test 1', () => {
+    numbers.push(3)
+  })
+  test('test 2', () => {
+    numbers.push(4)
+  })
+  test('test 3', () => {
+    numbers.push(5)
+  })
+
+  afterAll(() => {
+    expect(numbers).toStrictEqual([4, 5, 3, 1, 2])
+  })
+})