From cbe0ac1b7677df0f856b09e30180ec7916752c6b Mon Sep 17 00:00:00 2001 From: Josh <81541956+jhwang98@users.noreply.github.com> Date: Thu, 23 Feb 2023 23:19:32 +1100 Subject: [PATCH] jest-circus runs children in shuffled order (#12922) --- CHANGELOG.md | 1 + docs/CLI.md | 16 +++ docs/Configuration.md | 6 + .../__snapshots__/randomize.test.ts.snap | 89 +++++++++++++++ e2e/__tests__/randomize.test.ts | 85 ++++++++++++++ e2e/__tests__/showSeed.test.ts | 50 +++++---- e2e/jest-object/randomize-config.json | 4 + e2e/jest-object/showSeed-config.json | 4 + .../__snapshots__/snapshots.test.js.snap | 31 +++++ e2e/randomize/__tests__/each.test.js | 28 +++++ e2e/randomize/__tests__/hooks.test.js | 98 ++++++++++++++++ e2e/randomize/__tests__/snapshots.test.js | 28 +++++ e2e/randomize/__tests__/success.test.js | 74 ++++++++++++ .../different-config.json | 2 +- e2e/randomize/package.json | 5 + packages/jest-circus/package.json | 1 + .../jest-circus/src/__mocks__/testUtils.ts | 9 +- .../__snapshots__/randomizeTest.test.ts.snap | 85 ++++++++++++++ .../__snapshots__/shuffleArray.test.ts.snap | 106 ++++++++++++++++++ .../src/__tests__/randomizeTest.test.ts | 53 +++++++++ .../src/__tests__/shuffleArray.test.ts | 37 ++++++ .../jestAdapterInit.ts | 3 + packages/jest-circus/src/run.ts | 12 +- packages/jest-circus/src/shuffleArray.ts | 39 +++++++ packages/jest-circus/src/state.ts | 1 + packages/jest-cli/src/args.ts | 5 + packages/jest-config/src/ValidConfig.ts | 1 + .../src/__tests__/normalize.test.ts | 34 +++++- packages/jest-config/src/index.ts | 1 + packages/jest-config/src/normalize.ts | 10 +- packages/jest-types/src/Circus.ts | 2 + packages/jest-types/src/Config.ts | 3 + yarn.lock | 1 + 33 files changed, 889 insertions(+), 35 deletions(-) create mode 100644 e2e/__tests__/__snapshots__/randomize.test.ts.snap create mode 100644 e2e/__tests__/randomize.test.ts create mode 100644 e2e/jest-object/randomize-config.json create mode 100644 e2e/jest-object/showSeed-config.json create mode 100644 e2e/randomize/__tests__/__snapshots__/snapshots.test.js.snap create mode 100644 e2e/randomize/__tests__/each.test.js create mode 100644 e2e/randomize/__tests__/hooks.test.js create mode 100644 e2e/randomize/__tests__/snapshots.test.js create mode 100644 e2e/randomize/__tests__/success.test.js rename e2e/{jest-object => randomize}/different-config.json (75%) create mode 100644 e2e/randomize/package.json create mode 100644 packages/jest-circus/src/__tests__/__snapshots__/randomizeTest.test.ts.snap create mode 100644 packages/jest-circus/src/__tests__/__snapshots__/shuffleArray.test.ts.snap create mode 100644 packages/jest-circus/src/__tests__/randomizeTest.test.ts create mode 100644 packages/jest-circus/src/__tests__/shuffleArray.test.ts create mode 100644 packages/jest-circus/src/shuffleArray.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5961b507e514..803c728a0873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Features - `[jest-changed-files]` Support Sapling ([#13941](https://github.com/facebook/jest/pull/13941)) +- `[jest-circus, @jest/cli, jest-config]` Add feature to randomize order of tests via CLI flag or through the config file([#12922](https://github.com/facebook/jest/pull/12922)) - `[jest-cli, jest-config, @jest/core, jest-haste-map, @jest/reporters, jest-runner, jest-runtime, @jest/types]` Add `workerThreads` configuration option to allow using [worker threads](https://nodejs.org/dist/latest/docs/api/worker_threads.html) for parallelization ([#13939](https://github.com/facebook/jest/pull/13939)) - `[jest-config]` Add `openHandlesTimeout` option to configure possible open handles warning. ([#13875](https://github.com/facebook/jest/pull/13875)) - `[@jest/create-cache-key-function]` Allow passing `length` argument to `createCacheKey()` function and set its default value to `16` on Windows ([#13827](https://github.com/facebook/jest/pull/13827)) diff --git a/docs/CLI.md b/docs/CLI.md index a29ff2f6715b..006980433554 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -332,6 +332,22 @@ If configuration files are found in the specified paths, _all_ projects specifie ::: +### `--randomize` + +Shuffle the order of the tests within a file. The shuffling is based on the seed. See [`--seed=`](#--seednum) for more info. + +Seed value is displayed when this option is set. Equivalent to setting the CLI option [`--showSeed`](#--showseed). + +```bash +jest --randomize --seed 1234 +``` + +:::note + +This option is only supported using the default `jest-circus` test runner. + +::: + ### `--reporters` Run tests with specified reporters. [Reporter options](configuration#reporters-arraymodulename--modulename-options) are not available via CLI. Example with multiple reporters: diff --git a/docs/Configuration.md b/docs/Configuration.md index 3a4c66ba72dc..342de15478b9 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1222,6 +1222,12 @@ With the `projects` option enabled, Jest will copy the root-level configuration ::: +### `randomize` \[boolean] + +Default: `false` + +The equivalent of the [`--randomize`](CLI.md#--randomize) flag to randomize the order of the tests in a file. + ### `reporters` \[array<moduleName | \[moduleName, options]>] Default: `undefined` diff --git a/e2e/__tests__/__snapshots__/randomize.test.ts.snap b/e2e/__tests__/__snapshots__/randomize.test.ts.snap new file mode 100644 index 000000000000..7b9e9f889c89 --- /dev/null +++ b/e2e/__tests__/__snapshots__/randomize.test.ts.snap @@ -0,0 +1,89 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`works with each 1`] = ` +" ✓ test1 + ✓ test2 + ✓ test3 + describe2 + ✓ test4 + ✓ test6 + ✓ test5 + describe1 + ✓ test4 + ✓ test6 + ✓ test5 + describe3 + ✓ test11 + ✓ test12 + ✓ test10 + describe4 + ✓ test14 + ✓ test15 + ✓ test13" +`; + +exports[`works with hooks 1`] = ` +" ✓ test1 + ✓ test2 + ✓ test3 + describe2 + ✓ test7 + ✓ test9 + ✓ test8 + describe1 + ✓ test4 + ✓ test6 + ✓ test5 + describe3 + ✓ test11 + ✓ test12 + ✓ test10 + describe4 + ✓ test14 + ✓ test15 + ✓ test13" +`; + +exports[`works with passing tests 1`] = ` +" ✓ test1 + ✓ test2 + ✓ test3 + describe2 + ✓ test7 + ✓ test9 + ✓ test8 + describe1 + ✓ test4 + ✓ test6 + ✓ test5 + describe3 + ✓ test11 + ✓ test12 + ✓ test10 + describe4 + ✓ test14 + ✓ test15 + ✓ test13" +`; + +exports[`works with snapshots 1`] = ` +" ✓ test1 + ✓ test2 + ✓ test3 + describe2 + ✓ test4 + ✓ test6 + ✓ test5 + describe1 + ✓ test4 + ✓ test6 + ✓ test5 + describe3 + ✓ test11 + ✓ test12 + ✓ test10 + describe4 + ✓ test14 + ✓ test15 + ✓ test13" +`; diff --git a/e2e/__tests__/randomize.test.ts b/e2e/__tests__/randomize.test.ts new file mode 100644 index 000000000000..b0b83fd15c7e --- /dev/null +++ b/e2e/__tests__/randomize.test.ts @@ -0,0 +1,85 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import * as path from 'path'; +import {skipSuiteOnJasmine} from '@jest/test-utils'; +import {extractSummary} from '../Utils'; +import runJest, {RunJestResult} from '../runJest'; + +skipSuiteOnJasmine(); + +const dir = path.resolve(__dirname, '../randomize'); + +const trimFirstLine = (str: string): string => + str.split('\n').slice(1).join('\n'); + +function runJestTwice( + dir: string, + args: Array, +): [RunJestResult, RunJestResult] { + return [ + runJest(dir, [...args, '--randomize']), + runJest(dir, [...args, '--config', 'different-config.json']), + ]; +} + +test('works with passing tests', () => { + const [result1, result2] = runJestTwice(dir, [ + 'success.test.js', + '--seed', + '123', + ]); + + const rest1 = trimFirstLine(extractSummary(result1.stderr).rest); + const rest2 = trimFirstLine(extractSummary(result2.stderr).rest); + + expect(rest1).toEqual(rest2); + expect(rest1).toMatchSnapshot(); +}); + +test('works with each', () => { + const [result1, result2] = runJestTwice(dir, [ + 'each.test.js', + '--seed', + '123', + ]); + + const rest1 = trimFirstLine(extractSummary(result1.stderr).rest); + const rest2 = trimFirstLine(extractSummary(result2.stderr).rest); + + expect(rest1).toEqual(rest2); + expect(rest1).toMatchSnapshot(); +}); + +test('works with hooks', () => { + const [result1, result2] = runJestTwice(dir, [ + 'hooks.test.js', + '--seed', + '123', + ]); + + // Change in formatting could change this one + const rest1 = trimFirstLine(extractSummary(result1.stderr).rest); + const rest2 = trimFirstLine(extractSummary(result2.stderr).rest); + + expect(rest1).toEqual(rest2); + expect(rest1).toMatchSnapshot(); +}); + +test('works with snapshots', () => { + const [result1, result2] = runJestTwice(dir, [ + 'snapshots.test.js', + '--seed', + '123', + ]); + + const rest1 = trimFirstLine(extractSummary(result1.stderr).rest); + const rest2 = trimFirstLine(extractSummary(result2.stderr).rest); + + expect(rest1).toEqual(rest2); + expect(rest1).toMatchSnapshot(); +}); diff --git a/e2e/__tests__/showSeed.test.ts b/e2e/__tests__/showSeed.test.ts index 42b5c3d9be42..dba959d0325f 100644 --- a/e2e/__tests__/showSeed.test.ts +++ b/e2e/__tests__/showSeed.test.ts @@ -14,39 +14,41 @@ const dir = path.resolve(__dirname, '../jest-object'); const randomSeedValueRegExp = /Seed:\s+<>/; const seedValueRegExp = /Seed:\s+1234/; -test('--showSeed changes report to output seed', () => { - const {stderr} = runJest(dir, ['--showSeed', '--no-cache']); +describe.each(['showSeed', 'randomize'])('Option %s', option => { + test(`--${option} changes report to output seed`, () => { + const {stderr} = runJest(dir, [`--${option}`, '--no-cache']); - const {summary} = extractSummary(stderr); + const {summary} = extractSummary(stderr); - expect(replaceSeed(summary)).toMatch(randomSeedValueRegExp); -}); + expect(replaceSeed(summary)).toMatch(randomSeedValueRegExp); + }); -test('if --showSeed is not present the report will not show the seed', () => { - const {stderr} = runJest(dir, ['--seed', '1234']); + test(`if --${option} is not present the report will not show the seed`, () => { + const {stderr} = runJest(dir, ['--seed', '1234']); - const {summary} = extractSummary(stderr); + const {summary} = extractSummary(stderr); - expect(replaceSeed(summary)).not.toMatch(randomSeedValueRegExp); -}); + expect(replaceSeed(summary)).not.toMatch(randomSeedValueRegExp); + }); -test('if showSeed is present in the config the report will show the seed', () => { - const {stderr} = runJest(dir, [ - '--seed', - '1234', - '--config', - 'different-config.json', - ]); + test(`if ${option} is present in the config the report will show the seed`, () => { + const {stderr} = runJest(dir, [ + '--seed', + '1234', + '--config', + `${option}-config.json`, + ]); - const {summary} = extractSummary(stderr); + const {summary} = extractSummary(stderr); - expect(summary).toMatch(seedValueRegExp); -}); + expect(summary).toMatch(seedValueRegExp); + }); -test('--seed --showSeed will show the seed in the report', () => { - const {stderr} = runJest(dir, ['--showSeed', '--seed', '1234']); + test(`--seed --${option} will show the seed in the report`, () => { + const {stderr} = runJest(dir, [`--${option}`, '--seed', '1234']); - const {summary} = extractSummary(stderr); + const {summary} = extractSummary(stderr); - expect(summary).toMatch(seedValueRegExp); + expect(summary).toMatch(seedValueRegExp); + }); }); diff --git a/e2e/jest-object/randomize-config.json b/e2e/jest-object/randomize-config.json new file mode 100644 index 000000000000..434221a0bb3d --- /dev/null +++ b/e2e/jest-object/randomize-config.json @@ -0,0 +1,4 @@ +{ + "displayName": "Config from randomize-config.json file", + "randomize": true +} diff --git a/e2e/jest-object/showSeed-config.json b/e2e/jest-object/showSeed-config.json new file mode 100644 index 000000000000..a25af4e8d2b4 --- /dev/null +++ b/e2e/jest-object/showSeed-config.json @@ -0,0 +1,4 @@ +{ + "displayName": "Config from showSeed-config.json file", + "showSeed": true +} diff --git a/e2e/randomize/__tests__/__snapshots__/snapshots.test.js.snap b/e2e/randomize/__tests__/__snapshots__/snapshots.test.js.snap new file mode 100644 index 000000000000..8fb29cd59a98 --- /dev/null +++ b/e2e/randomize/__tests__/__snapshots__/snapshots.test.js.snap @@ -0,0 +1,31 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`describe1 test4 1`] = `4`; + +exports[`describe1 test5 1`] = `5`; + +exports[`describe1 test6 1`] = `6`; + +exports[`describe2 test4 1`] = `4`; + +exports[`describe2 test5 1`] = `5`; + +exports[`describe2 test6 1`] = `6`; + +exports[`describe3 describe4 test13 1`] = `13`; + +exports[`describe3 describe4 test14 1`] = `14`; + +exports[`describe3 describe4 test15 1`] = `15`; + +exports[`describe3 test10 1`] = `10`; + +exports[`describe3 test11 1`] = `11`; + +exports[`describe3 test12 1`] = `12`; + +exports[`test1 1`] = `1`; + +exports[`test2 1`] = `2`; + +exports[`test3 1`] = `3`; diff --git a/e2e/randomize/__tests__/each.test.js b/e2e/randomize/__tests__/each.test.js new file mode 100644 index 000000000000..9a065728417f --- /dev/null +++ b/e2e/randomize/__tests__/each.test.js @@ -0,0 +1,28 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +it.each([1, 2, 3])('test%d', () => { + expect(true).toBe(true); +}); + +describe.each([1, 2])('describe%d', () => { + it.each([4, 5, 6])('test%d', () => { + expect(true).toBe(true); + }); +}); + +describe('describe3', () => { + it.each([10, 11, 12])('test%d', () => { + expect(true).toBe(true); + }); + + describe('describe4', () => { + it.each([13, 14, 15])('test%d', () => { + expect(true).toBe(true); + }); + }); +}); diff --git a/e2e/randomize/__tests__/hooks.test.js b/e2e/randomize/__tests__/hooks.test.js new file mode 100644 index 000000000000..96bc057110d5 --- /dev/null +++ b/e2e/randomize/__tests__/hooks.test.js @@ -0,0 +1,98 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +beforeAll(() => { + process.stdout.write('This is before all\n'); +}); + +beforeEach(() => { + process.stdout.write('This is before each\n'); +}); + +afterEach(() => { + process.stdout.write('This is after each\n'); +}); + +afterAll(() => { + process.stdout.write('This is after all\n'); +}); + +it('test1', () => { + process.stdout.write('test1\n'); +}); + +it('test2', () => { + process.stdout.write('test2\n'); +}); + +it('test3', () => { + process.stdout.write('test3\n'); +}); + +describe('describe1', () => { + it('test4', () => { + process.stdout.write('test4\n'); + }); + + it('test5', () => { + process.stdout.write('test5\n'); + }); + + it('test6', () => { + process.stdout.write('test6\n'); + }); +}); + +describe('describe2', () => { + afterAll(() => { + process.stdout.write('This is after all describe2\n'); + }); + + it('test7', () => { + process.stdout.write('test7\n'); + }); + + it('test8', () => { + process.stdout.write('test8\n'); + }); + + it('test9', () => { + process.stdout.write('test9\n'); + }); +}); + +describe('describe3', () => { + beforeEach(() => { + process.stdout.write('This is before each describe3\n'); + }); + + it('test10', () => { + process.stdout.write('test10\n'); + }); + + it('test11', () => { + process.stdout.write('test11\n'); + }); + + it('test12', () => { + process.stdout.write('test12\n'); + }); + + describe('describe4', () => { + it('test13', () => { + process.stdout.write('test13\n'); + }); + + it('test14', () => { + process.stdout.write('test14\n'); + }); + + it('test15', () => { + process.stdout.write('test15\n'); + }); + }); +}); diff --git a/e2e/randomize/__tests__/snapshots.test.js b/e2e/randomize/__tests__/snapshots.test.js new file mode 100644 index 000000000000..dbaf12e835aa --- /dev/null +++ b/e2e/randomize/__tests__/snapshots.test.js @@ -0,0 +1,28 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +it.each([1, 2, 3])('test%d', n => { + expect(n).toMatchSnapshot(); +}); + +describe.each([1, 2])('describe%d', () => { + it.each([4, 5, 6])('test%d', n => { + expect(n).toMatchSnapshot(); + }); +}); + +describe('describe3', () => { + it.each([10, 11, 12])('test%d', n => { + expect(n).toMatchSnapshot(); + }); + + describe('describe4', () => { + it.each([13, 14, 15])('test%d', n => { + expect(n).toMatchSnapshot(); + }); + }); +}); diff --git a/e2e/randomize/__tests__/success.test.js b/e2e/randomize/__tests__/success.test.js new file mode 100644 index 000000000000..184b23cce28a --- /dev/null +++ b/e2e/randomize/__tests__/success.test.js @@ -0,0 +1,74 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +it('test1', () => { + expect(true).toBe(true); +}); + +it('test2', () => { + expect(true).toBe(true); +}); + +it('test3', () => { + expect(true).toBe(true); +}); + +describe('describe1', () => { + it('test4', () => { + expect(true).toBe(true); + }); + + it('test5', () => { + expect(true).toBe(true); + }); + + it('test6', () => { + expect(true).toBe(true); + }); +}); + +describe('describe2', () => { + it('test7', () => { + expect(true).toBe(true); + }); + + it('test8', () => { + expect(true).toBe(true); + }); + + it('test9', () => { + expect(true).toBe(true); + }); +}); + +describe('describe3', () => { + it('test10', () => { + expect(true).toBe(true); + }); + + it('test11', () => { + expect(true).toBe(true); + }); + + it('test12', () => { + expect(true).toBe(true); + }); + + describe('describe4', () => { + it('test13', () => { + expect(true).toBe(true); + }); + + it('test14', () => { + expect(true).toBe(true); + }); + + it('test15', () => { + expect(true).toBe(true); + }); + }); +}); diff --git a/e2e/jest-object/different-config.json b/e2e/randomize/different-config.json similarity index 75% rename from e2e/jest-object/different-config.json rename to e2e/randomize/different-config.json index ae9b2166eaff..4f8448990517 100644 --- a/e2e/jest-object/different-config.json +++ b/e2e/randomize/different-config.json @@ -1,4 +1,4 @@ { "displayName": "Config from different-config.json file", - "showSeed": true + "randomize": true } diff --git a/e2e/randomize/package.json b/e2e/randomize/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/randomize/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/packages/jest-circus/package.json b/packages/jest-circus/package.json index 8e98130e0d87..4bdff5cc2628 100644 --- a/packages/jest-circus/package.json +++ b/packages/jest-circus/package.json @@ -35,6 +35,7 @@ "jest-util": "workspace:^", "p-limit": "^3.1.0", "pretty-format": "workspace:^", + "pure-rand": "^6.0.0", "slash": "^3.0.0", "stack-utils": "^2.0.3" }, diff --git a/packages/jest-circus/src/__mocks__/testUtils.ts b/packages/jest-circus/src/__mocks__/testUtils.ts index 5bd876d4b604..358c0bf8b685 100644 --- a/packages/jest-circus/src/__mocks__/testUtils.ts +++ b/packages/jest-circus/src/__mocks__/testUtils.ts @@ -26,7 +26,10 @@ interface Result extends ExecaSyncReturnValue { error: string; } -export const runTest = (source: string) => { +export const runTest = ( + source: string, + opts?: {seed?: number; randomize?: boolean}, +) => { const filename = createHash('sha1') .update(source) .digest('hex') @@ -44,7 +47,9 @@ export const runTest = (source: string) => { global.afterAll = circus.afterAll; const testEventHandler = require('${TEST_EVENT_HANDLER_PATH}').default; - const addEventHandler = require('${CIRCUS_STATE_PATH}').addEventHandler; + const {addEventHandler, getState} = require('${CIRCUS_STATE_PATH}'); + getState().randomize = ${opts?.randomize}; + getState().seed = ${opts?.seed ?? 0}; addEventHandler(testEventHandler); ${source}; diff --git a/packages/jest-circus/src/__tests__/__snapshots__/randomizeTest.test.ts.snap b/packages/jest-circus/src/__tests__/__snapshots__/randomizeTest.test.ts.snap new file mode 100644 index 000000000000..f44411f9ac58 --- /dev/null +++ b/packages/jest-circus/src/__tests__/__snapshots__/randomizeTest.test.ts.snap @@ -0,0 +1,85 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`failures 1`] = ` +"start_describe_definition: describe +add_hook: beforeEach +add_hook: afterEach +add_test: one +add_test: two +finish_describe_definition: describe +run_start +run_describe_start: ROOT_DESCRIBE_BLOCK +run_describe_start: describe +test_start: one +hook_start: beforeEach +hook_success: beforeEach +test_fn_start: one +test_fn_failure: one +hook_start: afterEach +hook_failure: afterEach +test_done: one +test_start: two +hook_start: beforeEach +hook_success: beforeEach +test_fn_start: two +test_fn_success: two +hook_start: afterEach +hook_failure: afterEach +test_done: two +run_describe_finish: describe +run_describe_finish: ROOT_DESCRIBE_BLOCK +run_finish + +unhandledErrors: 0" +`; + +exports[`function descriptors 1`] = ` +"start_describe_definition: describer +add_test: One +finish_describe_definition: describer +run_start +run_describe_start: ROOT_DESCRIBE_BLOCK +run_describe_start: describer +test_start: One +test_fn_start: One +test_fn_success: One +test_done: One +run_describe_finish: describer +run_describe_finish: ROOT_DESCRIBE_BLOCK +run_finish + +unhandledErrors: 0" +`; + +exports[`simple test 1`] = ` +"start_describe_definition: describe +add_hook: beforeEach +add_hook: afterEach +add_test: one +add_test: two +finish_describe_definition: describe +run_start +run_describe_start: ROOT_DESCRIBE_BLOCK +run_describe_start: describe +test_start: one +hook_start: beforeEach +hook_success: beforeEach +test_fn_start: one +test_fn_success: one +hook_start: afterEach +hook_success: afterEach +test_done: one +test_start: two +hook_start: beforeEach +hook_success: beforeEach +test_fn_start: two +test_fn_success: two +hook_start: afterEach +hook_success: afterEach +test_done: two +run_describe_finish: describe +run_describe_finish: ROOT_DESCRIBE_BLOCK +run_finish + +unhandledErrors: 0" +`; diff --git a/packages/jest-circus/src/__tests__/__snapshots__/shuffleArray.test.ts.snap b/packages/jest-circus/src/__tests__/__snapshots__/shuffleArray.test.ts.snap new file mode 100644 index 000000000000..fa97936e90c3 --- /dev/null +++ b/packages/jest-circus/src/__tests__/__snapshots__/shuffleArray.test.ts.snap @@ -0,0 +1,106 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`rngBuilder creates a randomizer given seed 1 1`] = ` +Array [ + 0, + 2, + 4, + 0, + 2, + 8, + 5, + 9, + 9, + 5, +] +`; + +exports[`rngBuilder creates a randomizer given seed 2 1`] = ` +Array [ + 10, + 1, + 0, + 7, + 4, + 4, + 5, + 0, + 10, + 3, +] +`; + +exports[`rngBuilder creates a randomizer given seed 4 1`] = ` +Array [ + 8, + 10, + 3, + 2, + 5, + 2, + 3, + 4, + 8, + 5, +] +`; + +exports[`rngBuilder creates a randomizer given seed 8 1`] = ` +Array [ + 4, + 6, + 0, + 5, + 10, + 0, + 3, + 9, + 5, + 6, +] +`; + +exports[`rngBuilder creates a randomizer given seed 16 1`] = ` +Array [ + 7, + 9, + 3, + 2, + 8, + 1, + 6, + 1, + 10, + 1, +] +`; + +exports[`shuffleArray shuffles list ["a", "b", "c", "d"] 1`] = ` +Array [ + "c", + "b", + "a", + "d", +] +`; + +exports[`shuffleArray shuffles list ["a", "b", "c"] 1`] = ` +Array [ + "b", + "a", + "c", +] +`; + +exports[`shuffleArray shuffles list ["a", "b"] 1`] = ` +Array [ + "a", + "b", +] +`; + +exports[`shuffleArray shuffles list ["a"] 1`] = ` +Array [ + "a", +] +`; diff --git a/packages/jest-circus/src/__tests__/randomizeTest.test.ts b/packages/jest-circus/src/__tests__/randomizeTest.test.ts new file mode 100644 index 000000000000..a4035972d6f8 --- /dev/null +++ b/packages/jest-circus/src/__tests__/randomizeTest.test.ts @@ -0,0 +1,53 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {runTest} from '../__mocks__/testUtils'; + +test('simple test', () => { + const {stdout} = runTest( + ` + describe('describe', () => { + beforeEach(() => {}); + afterEach(() => {}); + test('one', () => {}); + test('two', () => {}); + }) + `, + {randomize: true, seed: 3}, + ); + + expect(stdout).toMatchSnapshot(); +}); + +test('function descriptors', () => { + const {stdout} = runTest( + ` + describe(function describer() {}, () => { + test(class One {}, () => {}); + }) + `, + {randomize: true, seed: 3}, + ); + + expect(stdout).toMatchSnapshot(); +}); + +test('failures', () => { + const {stdout} = runTest( + ` + describe('describe', () => { + beforeEach(() => {}); + afterEach(() => { throw new Error('banana')}); + test('one', () => { throw new Error('kentucky')}); + test('two', () => {}); + }) + `, + {randomize: true, seed: 3}, + ); + + expect(stdout).toMatchSnapshot(); +}); diff --git a/packages/jest-circus/src/__tests__/shuffleArray.test.ts b/packages/jest-circus/src/__tests__/shuffleArray.test.ts new file mode 100644 index 000000000000..b16ebb148177 --- /dev/null +++ b/packages/jest-circus/src/__tests__/shuffleArray.test.ts @@ -0,0 +1,37 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import shuffleArray, {rngBuilder} from '../shuffleArray'; + +describe('rngBuilder', () => { + // Breaking these orders would be a breaking change + // Some people will be using seeds relying on a particular order + test.each([1, 2, 4, 8, 16])('creates a randomizer given seed %s', seed => { + const rng = rngBuilder(seed); + const results = Array(10) + .fill(0) + .map(() => rng.next(0, 10)); + expect(results).toMatchSnapshot(); + }); +}); + +describe('shuffleArray', () => { + test('empty array is shuffled', () => { + const shuffled = shuffleArray([], rngBuilder(seed)); + expect(shuffled).toEqual([]); + }); + + // Breaking these orders would be a breaking change + // Some people will be using seeds relying on a particular order + const seed = 321; + test.each([[['a']], [['a', 'b']], [['a', 'b', 'c']], [['a', 'b', 'c', 'd']]])( + 'shuffles list %p', + l => { + expect(shuffleArray(l, rngBuilder(seed))).toMatchSnapshot(); + }, + ); +}); diff --git a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts index da0e51ce6d98..f9ac8076bd55 100644 --- a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts +++ b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts @@ -63,6 +63,9 @@ export const initialize = async ({ } getRunnerState().maxConcurrency = globalConfig.maxConcurrency; + getRunnerState().randomize = globalConfig.randomize; + getRunnerState().seed = globalConfig.seed; + // @ts-expect-error: missing `concurrent` which is added later const globalsObject: Global.TestFrameworkGlobals = { ...globals, diff --git a/packages/jest-circus/src/run.ts b/packages/jest-circus/src/run.ts index d7d7cc225704..7ccaf2d2e187 100644 --- a/packages/jest-circus/src/run.ts +++ b/packages/jest-circus/src/run.ts @@ -7,6 +7,7 @@ import pLimit = require('p-limit'); import type {Circus} from '@jest/types'; +import shuffleArray, {RandomNumberGenerator, rngBuilder} from './shuffleArray'; import {dispatch, getState} from './state'; import {RETRY_TIMES} from './types'; import { @@ -19,9 +20,10 @@ import { } from './utils'; const run = async (): Promise => { - const {rootDescribeBlock} = getState(); + const {rootDescribeBlock, seed, randomize} = getState(); + const rng = randomize ? rngBuilder(seed) : undefined; await dispatch({name: 'run_start'}); - await _runTestsForDescribeBlock(rootDescribeBlock, true); + await _runTestsForDescribeBlock(rootDescribeBlock, rng, true); await dispatch({name: 'run_finish'}); return makeRunResult( getState().rootDescribeBlock, @@ -31,6 +33,7 @@ const run = async (): Promise => { const _runTestsForDescribeBlock = async ( describeBlock: Circus.DescribeBlock, + rng: RandomNumberGenerator | undefined, isRootBlock = false, ) => { await dispatch({describeBlock, name: 'run_describe_start'}); @@ -68,10 +71,13 @@ const _runTestsForDescribeBlock = async ( const retryTimes = parseInt(global[RETRY_TIMES], 10) || 0; const deferredRetryTests = []; + if (rng) { + describeBlock.children = shuffleArray(describeBlock.children, rng); + } for (const child of describeBlock.children) { switch (child.type) { case 'describeBlock': { - await _runTestsForDescribeBlock(child); + await _runTestsForDescribeBlock(child, rng); break; } case 'test': { diff --git a/packages/jest-circus/src/shuffleArray.ts b/packages/jest-circus/src/shuffleArray.ts new file mode 100644 index 000000000000..e2afe6cdaaec --- /dev/null +++ b/packages/jest-circus/src/shuffleArray.ts @@ -0,0 +1,39 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {unsafeUniformIntDistribution, xoroshiro128plus} from 'pure-rand'; + +// Generates [from, to] inclusive +export type RandomNumberGenerator = { + next: (from: number, to: number) => number; +}; + +export const rngBuilder = (seed: number): RandomNumberGenerator => { + const gen = xoroshiro128plus(seed); + return {next: (from, to) => unsafeUniformIntDistribution(from, to, gen)}; +}; + +// Fisher-Yates shuffle +// This is performed in-place +export default function shuffleArray( + array: Array, + random: RandomNumberGenerator, +): Array { + const length = array.length; + if (length === 0) { + return []; + } + + for (let i = 0; i < length; i++) { + const n = random.next(i, length - 1); + const value = array[i]; + array[i] = array[n]; + array[n] = value; + } + + return array; +} diff --git a/packages/jest-circus/src/state.ts b/packages/jest-circus/src/state.ts index ab28c39118cb..0540dfd41116 100644 --- a/packages/jest-circus/src/state.ts +++ b/packages/jest-circus/src/state.ts @@ -30,6 +30,7 @@ const createState = (): Circus.State => { maxConcurrency: 5, parentProcess: null, rootDescribeBlock: ROOT_DESCRIBE_BLOCK, + seed: 0, testNamePattern: null, testTimeout: 5000, unhandledErrors: [], diff --git a/packages/jest-cli/src/args.ts b/packages/jest-cli/src/args.ts index 136ac0f562f0..ec62bdfc5a6b 100644 --- a/packages/jest-cli/src/args.ts +++ b/packages/jest-cli/src/args.ts @@ -455,6 +455,11 @@ export const options: {[key: string]: Options} = { string: true, type: 'array', }, + randomize: { + description: + 'Shuffle the order of the tests within a file. In order to choose the seed refer to the `--seed` CLI option.', + type: 'boolean', + }, reporters: { description: 'A list of custom reporters for the test suite.', string: true, diff --git a/packages/jest-config/src/ValidConfig.ts b/packages/jest-config/src/ValidConfig.ts index 461625146734..f6de3eac3338 100644 --- a/packages/jest-config/src/ValidConfig.ts +++ b/packages/jest-config/src/ValidConfig.ts @@ -121,6 +121,7 @@ export const initialOptions: Config.InitialOptions = { preset: 'react-native', prettierPath: '/node_modules/prettier', projects: ['project-a', 'project-b/'], + randomize: false, reporters: [ 'default', 'custom-reporter-1', diff --git a/packages/jest-config/src/__tests__/normalize.test.ts b/packages/jest-config/src/__tests__/normalize.test.ts index 26f751a95eb0..afbfe417a222 100644 --- a/packages/jest-config/src/__tests__/normalize.test.ts +++ b/packages/jest-config/src/__tests__/normalize.test.ts @@ -2150,14 +2150,14 @@ describe('seed', () => { seed: 2 ** 33, } as Config.Argv), ).rejects.toThrow( - 'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - is 8589934592', + 'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - instead it is 8589934592', ); await expect( normalize({rootDir: '/root/'}, { seed: -(2 ** 33), } as Config.Argv), ).rejects.toThrow( - 'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - is -8589934592', + 'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - instead it is -8589934592', ); }); }); @@ -2182,4 +2182,34 @@ describe('showSeed', () => { const {options} = await normalize({rootDir: '/root/'}, {} as Config.Argv); expect(options.showSeed).toBeFalsy(); }); + + test('showSeed is true when randomize is set', async () => { + const {options} = await normalize( + {randomize: true, rootDir: '/root/'}, + {} as Config.Argv, + ); + expect(options.showSeed).toBe(true); + }); +}); + +describe('randomize', () => { + test('randomize is set when argv flag is set', async () => { + const {options} = await normalize({rootDir: '/root/'}, { + randomize: true, + } as Config.Argv); + expect(options.randomize).toBe(true); + }); + + test('randomize is set when the config is set', async () => { + const {options} = await normalize( + {randomize: true, rootDir: '/root/'}, + {} as Config.Argv, + ); + expect(options.randomize).toBe(true); + }); + + test('randomize is false when neither is set', async () => { + const {options} = await normalize({rootDir: '/root/'}, {} as Config.Argv); + expect(options.randomize).toBeFalsy(); + }); }); diff --git a/packages/jest-config/src/index.ts b/packages/jest-config/src/index.ts index 1b8f6b47b63f..f68c5f19b0c0 100644 --- a/packages/jest-config/src/index.ts +++ b/packages/jest-config/src/index.ts @@ -117,6 +117,7 @@ const groupOptions = ( outputFile: options.outputFile, passWithNoTests: options.passWithNoTests, projects: options.projects, + randomize: options.randomize, replname: options.replname, reporters: options.reporters, rootDir: options.rootDir, diff --git a/packages/jest-config/src/normalize.ts b/packages/jest-config/src/normalize.ts index 41ede476b345..496d413deb3c 100644 --- a/packages/jest-config/src/normalize.ts +++ b/packages/jest-config/src/normalize.ts @@ -915,6 +915,7 @@ export default async function normalize( case 'openHandlesTimeout': case 'outputFile': case 'passWithNoTests': + case 'randomize': case 'replname': case 'resetMocks': case 'resetModules': @@ -1028,11 +1029,14 @@ export default async function normalize( newOptions.onlyChanged = newOptions.watch; } - newOptions.showSeed = newOptions.showSeed || argv.showSeed; + newOptions.randomize = newOptions.randomize || argv.randomize; + + newOptions.showSeed = + newOptions.randomize || newOptions.showSeed || argv.showSeed; const upperBoundSeedValue = 2 ** 31; - // xoroshiro128plus is used in v8 and is used here (at time of writing) + // bounds are determined by xoroshiro128plus which is used in v8 and is used here (at time of writing) newOptions.seed = argv.seed ?? Math.floor((2 ** 32 - 1) * Math.random() - upperBoundSeedValue); @@ -1042,7 +1046,7 @@ export default async function normalize( ) { throw new ValidationError( 'Validation Error', - `seed value must be between \`-0x80000000\` and \`0x7fffffff\` inclusive - is ${newOptions.seed}`, + `seed value must be between \`-0x80000000\` and \`0x7fffffff\` inclusive - instead it is ${newOptions.seed}`, ); } diff --git a/packages/jest-types/src/Circus.ts b/packages/jest-types/src/Circus.ts index 21f3a2bd1b8c..f7d1c0374923 100644 --- a/packages/jest-types/src/Circus.ts +++ b/packages/jest-types/src/Circus.ts @@ -215,7 +215,9 @@ export type State = { // the original ones. originalGlobalErrorHandlers?: GlobalErrorHandlers; parentProcess: Process | null; // process object from the outer scope + randomize?: boolean; rootDescribeBlock: DescribeBlock; + seed: number; testNamePattern?: RegExp | null; testTimeout: number; unhandledErrors: Array; diff --git a/packages/jest-types/src/Config.ts b/packages/jest-types/src/Config.ts index c37643178649..54240a747037 100644 --- a/packages/jest-types/src/Config.ts +++ b/packages/jest-types/src/Config.ts @@ -281,6 +281,7 @@ export type InitialOptions = Partial<{ preset: string | null | undefined; prettierPath: string | null | undefined; projects: Array; + randomize: boolean; replname: string | null | undefined; resetMocks: boolean; resetModules: boolean; @@ -396,6 +397,7 @@ export type GlobalConfig = { openHandlesTimeout: number; passWithNoTests: boolean; projects: Array; + randomize?: boolean; replname?: string; reporters?: Array; runTestsByPath: boolean; @@ -541,6 +543,7 @@ export type Argv = Arguments< preset: string | null | undefined; prettierPath: string | null | undefined; projects: Array; + randomize: boolean; reporters: Array; resetMocks: boolean; resetModules: boolean; diff --git a/yarn.lock b/yarn.lock index 1a2cbefc33f2..6ca858c85871 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12604,6 +12604,7 @@ __metadata: jest-util: "workspace:^" p-limit: ^3.1.0 pretty-format: "workspace:^" + pure-rand: ^6.0.0 slash: ^3.0.0 stack-utils: ^2.0.3 languageName: unknown