feat(config): add shuffle.files and shuffle.tests options (#5281)

Author: fenghan34

docs/config/index.md

@@ -1768,13 +1768,29 @@ Sharding is happening before sorting, and only if `--shard` option is provided.
 
 #### sequence.shuffle
 
-- **Type**: `boolean`
+- **Type**: `boolean | { files?, tests? }`
 - **Default**: `false`
 - **CLI**: `--sequence.shuffle`, `--sequence.shuffle=false`
 
-If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli).
+If you want files and tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli).
+
+Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your files and tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
+
+#### sequence.shuffle.files <Badge type="info">1.4.0+</Badge> {#sequence-shuffle-files}
+
+- **Type**: `boolean`
+- **Default**: `false`
+- **CLI**: `--sequence.shuffle.files`, `--sequence.shuffle.files=false`
+
+Whether to randomize files, be aware that long running tests will not start earlier if you enable this option.
+
+#### sequence.shuffle.tests <Badge type="info">1.4.0+</Badge> {#sequence-shuffle-tests}
+
+- **Type**: `boolean`
+- **Default**: `false`
+- **CLI**: `--sequence.shuffle.tests`, `--sequence.shuffle.tests=false`
 
-Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
+Whether to randomize tests.
 
 #### sequence.concurrent <Badge type="info">0.32.2+</Badge> {#sequence-concurrent}
 

packages/vitest/src/node/cli/cli-config.ts

@@ -436,7 +436,16 @@ export const cliOptionsConfig: VitestCLIOptions = {
     argument: '<options>',
     subcommands: {
       shuffle: {
-        description: 'Run tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)',
+        description: 'Run files and tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)',
+        argument: '',
+        subcommands: {
+          files: {
+            description: 'Run files in a random order. Long running tests will not start earlier if you enable this option. (default: false)',
+          },
+          tests: {
+            description: 'Run tests in a random oder (default: false)',
+          },
+        },
       },
       concurrent: {
         description: 'Make tests run in parallel (default: false)',

packages/vitest/src/node/config.ts

@@ -441,6 +441,11 @@ export function resolveConfig(
     resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir, resolved.name)
 
   resolved.sequence ??= {} as any
+  if (resolved.sequence.shuffle && typeof resolved.sequence.shuffle === 'object') {
+    const { files, tests } = resolved.sequence.shuffle
+    resolved.sequence.sequencer ??= files ? RandomSequencer : BaseSequencer
+    resolved.sequence.shuffle = tests
+  }
   if (!resolved.sequence?.sequencer) {
     // CLI flag has higher priority
     resolved.sequence.sequencer = resolved.sequence.shuffle

packages/vitest/src/types/config.ts

@@ -48,10 +48,22 @@ interface SequenceOptions {
    */
   sequencer?: TestSequencerConstructor
   /**
-   * Should tests run in random order.
+   * Should files and tests run in random order.
    * @default false
    */
-  shuffle?: boolean
+  shuffle?: boolean | {
+    /**
+     * Should files run in random order. Long running tests will not start
+     * earlier if you enable this option.
+     * @default false
+     */
+    files?: boolean
+    /**
+     * Should tests run in random order.
+     * @default false
+     */
+    tests?: boolean
+  }
   /**
    * Should tests run in parallel.
    * @default false

test/config/test/shuffle-options.test.ts

@@ -0,0 +1,49 @@
+import type { InlineConfig } from 'vitest'
+import { expect, test } from 'vitest'
+import { runVitest } from '../../test-utils'
+
+function run(sequence: InlineConfig['sequence']) {
+  return runVitest({
+    sequence,
+    include: [],
+  })
+}
+
+class CustomSequencer {
+  sort() {
+    return []
+  }
+
+  shard() {
+    return []
+  }
+}
+
+test.each([
+  false,
+  { files: false, tests: false },
+  { files: false, tests: true },
+],
+)('should use BaseSequencer if shuffle is %o', async (shuffle) => {
+  const { vitest } = await run({ shuffle })
+  expect(vitest?.config.sequence.sequencer.name).toBe('BaseSequencer')
+})
+
+test.each([
+  true,
+  { files: true, tests: false },
+  { files: true, tests: true },
+])('should use RandomSequencer if shuffle is %o', async (shuffle) => {
+  const { vitest } = await run({ shuffle })
+  expect(vitest?.config.sequence.sequencer.name).toBe('RandomSequencer')
+})
+
+test.each([
+  false,
+  true,
+  { files: true, tests: false },
+  { files: true, tests: true },
+])('should always use CustomSequencer if passed', async (shuffle) => {
+  const { vitest } = await run({ shuffle, sequencer: CustomSequencer })
+  expect(vitest?.config.sequence.sequencer.name).toBe('CustomSequencer')
+})

test/core/test/cli-test.test.ts

@@ -223,6 +223,13 @@ test('cache is parsed correctly', () => {
   })
 })
 
+test('shuffle is parsed correctly', () => {
+  expect(getCLIOptions('--sequence.shuffle')).toEqual({ sequence: { shuffle: true } })
+  expect(getCLIOptions('--sequence.shuffle=false')).toEqual({ sequence: { shuffle: false } })
+  expect(getCLIOptions('--sequence.shuffle.files --sequence.shuffle.tests')).toEqual({ sequence: { shuffle: { files: true, tests: true } } })
+  expect(getCLIOptions('--sequence.shuffle.files=false --sequence.shuffle.tests=false')).toEqual({ sequence: { shuffle: { files: false, tests: false } } })
+})
+
 test('typecheck correctly passes down arguments', () => {
   const { options, args } = parseArguments('--typecheck some.name.ts')
   expect(options).toEqual({ typecheck: { enabled: true } })