From 356db87b71c238585a9315881ef89544d19dc4d7 Mon Sep 17 00:00:00 2001 From: Han Date: Thu, 14 Mar 2024 22:50:03 +0800 Subject: [PATCH] feat(config): add `shuffle.files` and `shuffle.tests` options (#5281) --- docs/config/index.md | 22 ++++++++-- packages/vitest/src/node/cli/cli-config.ts | 11 ++++- packages/vitest/src/node/config.ts | 5 +++ packages/vitest/src/types/config.ts | 16 ++++++- test/config/test/shuffle-options.test.ts | 49 ++++++++++++++++++++++ test/core/test/cli-test.test.ts | 7 ++++ 6 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 test/config/test/shuffle-options.test.ts diff --git a/docs/config/index.md b/docs/config/index.md index b297d11c926f..e9bf7d5730d2 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1768,13 +1768,29 @@ Sharding is happening before sorting, and only if `--shard` option is provided. #### sequence.shuffle -- **Type**: `boolean` +- **Type**: `boolean | { files?, tests? }` - **Default**: `false` - **CLI**: `--sequence.shuffle`, `--sequence.shuffle=false` -If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). +If you want files and tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli). + +Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your files and tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. + +#### sequence.shuffle.files 1.4.0+ {#sequence-shuffle-files} + +- **Type**: `boolean` +- **Default**: `false` +- **CLI**: `--sequence.shuffle.files`, `--sequence.shuffle.files=false` + +Whether to randomize files, be aware that long running tests will not start earlier if you enable this option. + +#### sequence.shuffle.tests 1.4.0+ {#sequence-shuffle-tests} + +- **Type**: `boolean` +- **Default**: `false` +- **CLI**: `--sequence.shuffle.tests`, `--sequence.shuffle.tests=false` -Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously. +Whether to randomize tests. #### sequence.concurrent 0.32.2+ {#sequence-concurrent} diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 28e1b48047e0..c727d3a66276 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -436,7 +436,16 @@ export const cliOptionsConfig: VitestCLIOptions = { argument: '', subcommands: { shuffle: { - description: 'Run tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)', + description: 'Run files and tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)', + argument: '', + subcommands: { + files: { + description: 'Run files in a random order. Long running tests will not start earlier if you enable this option. (default: false)', + }, + tests: { + description: 'Run tests in a random oder (default: false)', + }, + }, }, concurrent: { description: 'Make tests run in parallel (default: false)', diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index e1e46d16c6af..6d054f0a417e 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -441,6 +441,11 @@ export function resolveConfig( resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir, resolved.name) resolved.sequence ??= {} as any + if (resolved.sequence.shuffle && typeof resolved.sequence.shuffle === 'object') { + const { files, tests } = resolved.sequence.shuffle + resolved.sequence.sequencer ??= files ? RandomSequencer : BaseSequencer + resolved.sequence.shuffle = tests + } if (!resolved.sequence?.sequencer) { // CLI flag has higher priority resolved.sequence.sequencer = resolved.sequence.shuffle diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 2c1e5951d3c5..5888f434be24 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -48,10 +48,22 @@ interface SequenceOptions { */ sequencer?: TestSequencerConstructor /** - * Should tests run in random order. + * Should files and tests run in random order. * @default false */ - shuffle?: boolean + shuffle?: boolean | { + /** + * Should files run in random order. Long running tests will not start + * earlier if you enable this option. + * @default false + */ + files?: boolean + /** + * Should tests run in random order. + * @default false + */ + tests?: boolean + } /** * Should tests run in parallel. * @default false diff --git a/test/config/test/shuffle-options.test.ts b/test/config/test/shuffle-options.test.ts new file mode 100644 index 000000000000..554a38863ccc --- /dev/null +++ b/test/config/test/shuffle-options.test.ts @@ -0,0 +1,49 @@ +import type { InlineConfig } from 'vitest' +import { expect, test } from 'vitest' +import { runVitest } from '../../test-utils' + +function run(sequence: InlineConfig['sequence']) { + return runVitest({ + sequence, + include: [], + }) +} + +class CustomSequencer { + sort() { + return [] + } + + shard() { + return [] + } +} + +test.each([ + false, + { files: false, tests: false }, + { files: false, tests: true }, +], +)('should use BaseSequencer if shuffle is %o', async (shuffle) => { + const { vitest } = await run({ shuffle }) + expect(vitest?.config.sequence.sequencer.name).toBe('BaseSequencer') +}) + +test.each([ + true, + { files: true, tests: false }, + { files: true, tests: true }, +])('should use RandomSequencer if shuffle is %o', async (shuffle) => { + const { vitest } = await run({ shuffle }) + expect(vitest?.config.sequence.sequencer.name).toBe('RandomSequencer') +}) + +test.each([ + false, + true, + { files: true, tests: false }, + { files: true, tests: true }, +])('should always use CustomSequencer if passed', async (shuffle) => { + const { vitest } = await run({ shuffle, sequencer: CustomSequencer }) + expect(vitest?.config.sequence.sequencer.name).toBe('CustomSequencer') +}) diff --git a/test/core/test/cli-test.test.ts b/test/core/test/cli-test.test.ts index 0cf5b61b3847..b40ff72f9efc 100644 --- a/test/core/test/cli-test.test.ts +++ b/test/core/test/cli-test.test.ts @@ -223,6 +223,13 @@ test('cache is parsed correctly', () => { }) }) +test('shuffle is parsed correctly', () => { + expect(getCLIOptions('--sequence.shuffle')).toEqual({ sequence: { shuffle: true } }) + expect(getCLIOptions('--sequence.shuffle=false')).toEqual({ sequence: { shuffle: false } }) + expect(getCLIOptions('--sequence.shuffle.files --sequence.shuffle.tests')).toEqual({ sequence: { shuffle: { files: true, tests: true } } }) + expect(getCLIOptions('--sequence.shuffle.files=false --sequence.shuffle.tests=false')).toEqual({ sequence: { shuffle: { files: false, tests: false } } }) +}) + test('typecheck correctly passes down arguments', () => { const { options, args } = parseArguments('--typecheck some.name.ts') expect(options).toEqual({ typecheck: { enabled: true } })