From 54eab578597125bf666b4d9ccfe58273c2934dc6 Mon Sep 17 00:00:00 2001 From: Mario Nebl Date: Sun, 6 Mar 2022 20:13:27 +1100 Subject: [PATCH] feat: implement `--shard` option (#12546) --- .circleci/config.yml | 6 +- .github/workflows/nodejs.yml | 19 ++- CHANGELOG.md | 1 + docs/CLI.md | 20 ++- docs/Configuration.md | 26 +++- docs/Troubleshooting.md | 12 ++ e2e/__tests__/shard.test.ts | 86 +++++++++++++ e2e/shard/__tests__/1.test.js | 13 ++ e2e/shard/__tests__/2.test.js | 13 ++ e2e/shard/__tests__/3.test.js | 13 ++ e2e/shard/no-sharding-test-sequencer.js | 11 ++ e2e/shard/package.json | 5 + e2e/shard/sharding-test-sequencer.js | 18 +++ .../jest-cli/src/__tests__/cli/args.test.ts | 56 ++++----- packages/jest-cli/src/cli/args.ts | 12 +- .../src/__tests__/normalize.test.ts | 10 ++ .../src/__tests__/parseShardPair.test.ts | 59 +++++++++ packages/jest-config/src/index.ts | 1 + packages/jest-config/src/normalize.ts | 5 + packages/jest-config/src/parseShardPair.ts | 43 +++++++ packages/jest-core/src/runJest.ts | 9 ++ packages/jest-test-sequencer/package.json | 4 +- ...quencer.test.js => test_sequencer.test.ts} | 116 +++++++++++++++--- packages/jest-test-sequencer/src/index.ts | 101 ++++++++++++--- packages/jest-test-sequencer/tsconfig.json | 3 +- packages/jest-types/src/Config.ts | 7 ++ yarn.lock | 2 + 27 files changed, 592 insertions(+), 79 deletions(-) create mode 100644 e2e/__tests__/shard.test.ts create mode 100644 e2e/shard/__tests__/1.test.js create mode 100644 e2e/shard/__tests__/2.test.js create mode 100644 e2e/shard/__tests__/3.test.js create mode 100644 e2e/shard/no-sharding-test-sequencer.js create mode 100644 e2e/shard/package.json create mode 100644 e2e/shard/sharding-test-sequencer.js create mode 100644 packages/jest-config/src/__tests__/parseShardPair.test.ts create mode 100644 packages/jest-config/src/parseShardPair.ts rename packages/jest-test-sequencer/src/__tests__/{test_sequencer.test.js => test_sequencer.test.ts} (72%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 450c196c3dce..f6614bab3220 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,26 +21,28 @@ jobs: type: string working_directory: ~/jest executor: node/default + parallelism: 4 steps: - checkout - node/install: node-version: << parameters.node-version >> - node/install-packages: *install - run: - command: yarn test-ci-partial + command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL - store_test_results: path: reports/junit test-jest-jasmine: working_directory: ~/jest executor: node/default + parallelism: 4 steps: - checkout - node/install: node-version: lts/* - node/install-packages: *install - run: - command: JEST_JASMINE=1 yarn test-ci-partial && JEST_JASMINE=1 yarn test-leak + command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL && JEST_JASMINE=1 yarn test-leak - store_test_results: path: reports/junit diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 50871be0f1ea..7fdbb93b150c 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -65,13 +65,15 @@ jobs: run: yarn lint:prettier:ci - name: check copyright headers run: yarn check-copyright-headers + test: - name: Node v${{ matrix.node-version }} on ${{ matrix.os }} + name: Node v${{ matrix.node-version }} on ${{ matrix.os }} (${{ matrix.shard }}) strategy: fail-fast: false matrix: node-version: [12.x, 14.x, 16.x, 17.x] os: [ubuntu-latest, macOS-latest, windows-latest] + shard: ['1/4', '2/4', '3/4', '4/4'] runs-on: ${{ matrix.os }} needs: prepare-yarn-cache @@ -96,14 +98,15 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests - run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-jasmine: - name: Node LTS on ${{ matrix.os }} using jest-jasmine2 + name: Node LTS on ${{ matrix.os }} using jest-jasmine2 (${{ matrix.shard }}) strategy: fail-fast: false matrix: os: [ubuntu-latest, macOS-latest, windows-latest] + shard: ['1/4', '2/4', '3/4', '4/4'] runs-on: ${{ matrix.os }} needs: prepare-yarn-cache @@ -128,10 +131,14 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine - run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-coverage: - name: Node LTS on Ubuntu with coverage + name: Node LTS on Ubuntu with coverage (${{ matrix.shard }}) + strategy: + fail-fast: false + matrix: + shard: ['1/4', '2/4', '3/4', '4/4'] runs-on: ubuntu-latest needs: prepare-yarn-cache @@ -151,7 +158,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests with coverage run: | - yarn jest-coverage --color --config jest.config.ci.js --max-workers ${{ steps.cpu-cores.outputs.count }} + yarn jest-coverage --color --config jest.config.ci.js --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} yarn test-leak - name: map coverage run: node ./scripts/mapCoverage.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 37f90b8d5235..1396884ddbb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - `[expect]` Expose `AsymmetricMatchers`, `MatcherFunction` and `MatcherFunctionWithState` interfaces ([#12363](https://github.com/facebook/jest/pull/12363), [#12376](https://github.com/facebook/jest/pull/12376)) - `[jest-circus, jest-jasmine2]` Allowed classes and functions as `describe` and `it`/`test` names ([#12484](https://github.com/facebook/jest/pull/12484)) - `[jest-cli, jest-config]` [**BREAKING**] Remove `testURL` config, use `testEnvironmentOptions.url` instead ([#10797](https://github.com/facebook/jest/pull/10797)) +- `[jest-cli, jest-core]` Add `--shard` parameter for distributed parallel test execution ([#12546](https://github.com/facebook/jest/pull/12546)) - `[jest-config]` [**BREAKING**] Stop shipping `jest-environment-jsdom` by default ([#12354](https://github.com/facebook/jest/pull/12354)) - `[jest-config]` [**BREAKING**] Stop shipping `jest-jasmine2` by default ([#12355](https://github.com/facebook/jest/pull/12355)) - `[jest-config, @jest/types]` Add `ci` to `GlobalConfig` ([#12378](https://github.com/facebook/jest/pull/12378)) diff --git a/docs/CLI.md b/docs/CLI.md index 75bfb2164491..d1cc0aecd2ae 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -338,6 +338,24 @@ Run only the tests of the specified projects. Jest uses the attribute `displayNa A list of paths to modules that run some code to configure or to set up the testing framework before each test. Beware that files imported by the setup scripts will not be mocked during testing. +### `--shard` + +The test suite shard to execute in a format of `(?\d+)/(?\d+)`. + +`shardIndex` describes which shard to select while `shardCount` controls the number of shards the suite should be split into. + +`shardIndex` and `shardCount` have to be 1-based, positive numbers, and `shardIndex` has to be lower than or equal to `shardCount`. + +When `shard` is specified the configured [`testSquencer`](Configuration.md#testsequencer-string) has to implement a `shard` method. + +For example, to split the suite into three shards, each running one third of the tests: + +``` +jest --shard=1/3 +jest --shard=2/3 +jest --shard=3/3 +``` + ### `--showConfig` Print your Jest config and then exits. @@ -389,7 +407,7 @@ Lets you specify a custom test runner. ### `--testSequencer=` -Lets you specify a custom test sequencer. Please refer to the documentation of the corresponding configuration property for details. +Lets you specify a custom test sequencer. Please refer to the [`testSequencer` configuration](Configuration.md#testsequencer-string) for details. ### `--testTimeout=` diff --git a/docs/Configuration.md b/docs/Configuration.md index ec574c4129c8..038a2917e832 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1327,7 +1327,13 @@ An example of such function can be found in our default [jasmine2 test runner pa Default: `@jest/test-sequencer` -This option allows you to use a custom sequencer instead of Jest's default. `sort` may optionally return a Promise. +This option allows you to use a custom sequencer instead of Jest's default. + +:::tip + +Both `sort` and `shard` may optionally return a `Promise`. + +::: Example: @@ -1337,6 +1343,24 @@ Sort test path alphabetically. const Sequencer = require('@jest/test-sequencer').default; class CustomSequencer extends Sequencer { + /** + * Select tests for shard requested via --shard=shardIndex/shardCount + * Sharding is applied before sorting + */ + shard(tests, {shardIndex, shardCount}) { + const shardSize = Math.ceil(tests.length / options.shardCount); + const shardStart = shardSize * (options.shardIndex - 1); + const shardEnd = shardSize * options.shardIndex; + + return [...tests] + .sort((a, b) => (a.path > b.path ? 1 : -1)) + .slice(shardStart, shardEnd); + } + + /** + * Sort test to determine order of execution + * Sorting is applied after sharding + */ sort(tests) { // Test structure information // https://github.com/facebook/jest/blob/6b8b1404a1d9254e7d5d90a8934087a9c9899dab/packages/jest-runner/src/types.ts#L17-L21 diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 1ecee62f5936..5c9782875500 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -182,6 +182,18 @@ jest --maxWorkers=4 yarn test --maxWorkers=4 ``` +If you use GitHub Actions, you can use [`github-actions-cpu-cores`](https://github.com/SimenB/github-actions-cpu-cores) to detect number of CPUs, and pass that to Jest. + +```yaml +- name: Get number of CPU cores + id: cpu-cores + uses: SimenB/github-actions-cpu-cores@v1 +- name: run tests + run: yarn jest --max-workers ${{ steps.cpu-cores.outputs.count }} +``` + +Another thing you can do is use the [`shard`](CLI.md#--shard) flag to parallelize the test run across multiple machines. + ## `coveragePathIgnorePatterns` seems to not have any effect. Make sure you are not using the `babel-plugin-istanbul` plugin. Jest wraps Istanbul, and therefore also tells Istanbul what files to instrument with coverage collection. When using `babel-plugin-istanbul`, every file that is processed by Babel will have coverage collection code, hence it is not being ignored by `coveragePathIgnorePatterns`. diff --git a/e2e/__tests__/shard.test.ts b/e2e/__tests__/shard.test.ts new file mode 100644 index 000000000000..568edc9815ef --- /dev/null +++ b/e2e/__tests__/shard.test.ts @@ -0,0 +1,86 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +import * as path from 'path'; +import runJest from '../runJest'; + +test('--shard=1/1', () => { + const result = runJest('shard', ['--shard=1/1', '--listTests']); + + const paths = result.stdout + .split('\n') + .filter(Boolean) + .map(file => path.basename(file)) + .sort(); + + expect(paths).toEqual(['1.test.js', '2.test.js', '3.test.js']); +}); + +test('--shard=1/2', () => { + const result = runJest('shard', ['--shard=1/2', '--listTests']); + + const paths = result.stdout + .split('\n') + .filter(Boolean) + .map(file => path.basename(file)) + .sort(); + + expect(paths).toEqual(['1.test.js', '3.test.js']); +}); + +test('--shard=2/2', () => { + const result = runJest('shard', ['--shard=2/2', '--listTests']); + + const paths = result.stdout + .split('\n') + .filter(Boolean) + .map(file => path.basename(file)); + + expect(paths).toEqual(['2.test.js']); +}); + +test('--shard=4/4', () => { + const result = runJest('shard', ['--shard=4/4', '--listTests']); + + const paths = result.stdout + .split('\n') + .filter(Boolean) + .map(file => path.basename(file)); + + // project only has 3 files + // shards > 3 are empty + expect(paths).toEqual([]); +}); + +test('--shard=1/2 custom non-sharding test sequencer', () => { + const result = runJest('shard', [ + '--shard=1/2', + '--listTests', + '--testSequencer=./no-sharding-test-sequencer.js', + ]); + + expect(result).toMatchObject({ + failed: true, + stderr: expect.stringMatching( + /Shard (.*) requested, but test sequencer (.*) in (.*) has no shard method./, + ), + }); +}); + +test('--shard=1/2 custom sharding test sequencer', () => { + const result = runJest('shard', [ + '--shard=1/2', + '--listTests', + '--testSequencer=./sharding-test-sequencer.js', + ]); + + const paths = result.stdout + .split('\n') + .filter(Boolean) + .map(file => path.basename(file)); + + expect(paths).toEqual(['3.test.js']); +}); diff --git a/e2e/shard/__tests__/1.test.js b/e2e/shard/__tests__/1.test.js new file mode 100644 index 000000000000..8362bdfa2248 --- /dev/null +++ b/e2e/shard/__tests__/1.test.js @@ -0,0 +1,13 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('1-1', () => expect(2).toBe(2)); + +test('1-2', () => expect(2).toBe(2)); + +test('1-3', () => expect(2).toBe(2)); diff --git a/e2e/shard/__tests__/2.test.js b/e2e/shard/__tests__/2.test.js new file mode 100644 index 000000000000..50faeecd1743 --- /dev/null +++ b/e2e/shard/__tests__/2.test.js @@ -0,0 +1,13 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('2-1', () => expect(2).toBe(2)); + +test('2-2', () => expect(2).toBe(2)); + +test('2-3', () => expect(2).toBe(2)); diff --git a/e2e/shard/__tests__/3.test.js b/e2e/shard/__tests__/3.test.js new file mode 100644 index 000000000000..1ddf79ff3a36 --- /dev/null +++ b/e2e/shard/__tests__/3.test.js @@ -0,0 +1,13 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('3-1', () => expect(2).toBe(2)); + +test('3-2', () => expect(2).toBe(2)); + +test('3-3', () => expect(2).toBe(2)); diff --git a/e2e/shard/no-sharding-test-sequencer.js b/e2e/shard/no-sharding-test-sequencer.js new file mode 100644 index 000000000000..10c40335432b --- /dev/null +++ b/e2e/shard/no-sharding-test-sequencer.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +module.exports = class NoShardingSequencer { + sort(tests) { + return tests; + } +}; diff --git a/e2e/shard/package.json b/e2e/shard/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/shard/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/e2e/shard/sharding-test-sequencer.js b/e2e/shard/sharding-test-sequencer.js new file mode 100644 index 000000000000..f3c38ba8f008 --- /dev/null +++ b/e2e/shard/sharding-test-sequencer.js @@ -0,0 +1,18 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +module.exports = class NoShardingSequencer { + shard(tests) { + return [ + Array.from(tests).sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0, + )[2], + ]; + } + sort(tests) { + return tests; + } +}; diff --git a/packages/jest-cli/src/__tests__/cli/args.test.ts b/packages/jest-cli/src/__tests__/cli/args.test.ts index ec302f70d734..f40f37aeb873 100644 --- a/packages/jest-cli/src/__tests__/cli/args.test.ts +++ b/packages/jest-cli/src/__tests__/cli/args.test.ts @@ -11,84 +11,78 @@ import {constants} from 'jest-config'; import {buildArgv} from '../../cli'; import {check} from '../../cli/args'; +const argv = (input: Partial): Config.Argv => input as Config.Argv; + describe('check', () => { it('returns true if the arguments are valid', () => { - const argv = {} as Config.Argv; - expect(check(argv)).toBe(true); + expect(check(argv({}))).toBe(true); }); it('raises an exception if runInBand and maxWorkers are both specified', () => { - const argv = {maxWorkers: 2, runInBand: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({maxWorkers: 2, runInBand: true}))).toThrow( 'Both --runInBand and --maxWorkers were specified', ); }); it('raises an exception if onlyChanged and watchAll are both specified', () => { - const argv = {onlyChanged: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({onlyChanged: true, watchAll: true}))).toThrow( 'Both --onlyChanged and --watchAll were specified', ); }); it('raises an exception if onlyFailures and watchAll are both specified', () => { - const argv = {onlyFailures: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({onlyFailures: true, watchAll: true}))).toThrow( 'Both --onlyFailures and --watchAll were specified', ); }); it('raises an exception when lastCommit and watchAll are both specified', () => { - const argv = {lastCommit: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({lastCommit: true, watchAll: true}))).toThrow( 'Both --lastCommit and --watchAll were specified', ); }); it('raises an exception if findRelatedTests is specified with no file paths', () => { - const argv = { - _: [] as Array, - findRelatedTests: true, - } as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => + check( + argv({ + _: [], + findRelatedTests: true, + }), + ), + ).toThrow( 'The --findRelatedTests option requires file paths to be specified', ); }); it('raises an exception if maxWorkers is specified with no number', () => { - const argv = {maxWorkers: undefined} as unknown as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({maxWorkers: undefined}))).toThrow( 'The --maxWorkers (-w) option requires a number or string to be specified', ); }); it('allows maxWorkers to be a %', () => { - const argv = {maxWorkers: '50%'} as unknown as Config.Argv; - expect(() => check(argv)).not.toThrow(); + expect(() => check(argv({maxWorkers: '50%'}))).not.toThrow(); }); test.each(constants.JEST_CONFIG_EXT_ORDER.map(e => e.substring(1)))( 'allows using "%s" file for --config option', ext => { + expect(() => check(argv({config: `jest.config.${ext}`}))).not.toThrow(); expect(() => - check({config: `jest.config.${ext}`} as Config.Argv), - ).not.toThrow(); - expect(() => - check({config: `../test/test/my_conf.${ext}`} as Config.Argv), + check(argv({config: `../test/test/my_conf.${ext}`})), ).not.toThrow(); }, ); it('raises an exception if selectProjects is not provided any project names', () => { - const argv: Config.Argv = {selectProjects: []} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({selectProjects: []}))).toThrow( 'The --selectProjects option requires the name of at least one project to be specified.\n', ); }); it('raises an exception if config is not a valid JSON string', () => { - const argv = {config: 'x:1'} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({config: 'x:1'}))).toThrow( 'The --config option requires a JSON string literal, or a file path with one of these extensions: .js, .ts, .mjs, .cjs, .json', ); }); @@ -97,12 +91,8 @@ describe('check', () => { const message = 'The --config option requires a JSON string literal, or a file path with one of these extensions: .js, .ts, .mjs, .cjs, .json'; - expect(() => check({config: 'jest.configjs'} as Config.Argv)).toThrow( - message, - ); - expect(() => check({config: 'jest.config.exe'} as Config.Argv)).toThrow( - message, - ); + expect(() => check(argv({config: 'jest.configjs'}))).toThrow(message); + expect(() => check(argv({config: 'jest.config.exe'}))).toThrow(message); }); }); diff --git a/packages/jest-cli/src/cli/args.ts b/packages/jest-cli/src/cli/args.ts index 92c13247f646..57c55a59f129 100644 --- a/packages/jest-cli/src/cli/args.ts +++ b/packages/jest-cli/src/cli/args.ts @@ -361,7 +361,7 @@ export const options = { description: 'An array of file extensions your modules use. If you ' + 'require modules without specifying a file extension, these are the ' + - 'extensions Jest will look for. ', + 'extensions Jest will look for.', string: true, type: 'array', }, @@ -510,17 +510,23 @@ export const options = { setupFiles: { description: 'A list of paths to modules that run some code to configure or ' + - 'set up the testing environment before each test. ', + 'set up the testing environment before each test.', string: true, type: 'array', }, setupFilesAfterEnv: { description: 'A list of paths to modules that run some code to configure or ' + - 'set up the testing framework before each test ', + 'set up the testing framework before each test', string: true, type: 'array', }, + shard: { + description: + 'Shard tests and execute only the selected shard, specify in ' + + 'the form "current/all". 1-based, for example "3/5".', + type: 'string', + }, showConfig: { description: 'Print your jest config and then exits.', type: 'boolean', diff --git a/packages/jest-config/src/__tests__/normalize.test.ts b/packages/jest-config/src/__tests__/normalize.test.ts index 7c96711a8c89..ccca146c1c54 100644 --- a/packages/jest-config/src/__tests__/normalize.test.ts +++ b/packages/jest-config/src/__tests__/normalize.test.ts @@ -1972,3 +1972,13 @@ describe('moduleLoader', () => { expect(console.warn).toMatchSnapshot(); }); }); + +describe('shards', () => { + it('should be object if defined', async () => { + const {options} = await normalize({rootDir: '/root/'}, { + shard: '1/2', + } as Config.Argv); + + expect(options.shard).toEqual({shardCount: 2, shardIndex: 1}); + }); +}); diff --git a/packages/jest-config/src/__tests__/parseShardPair.test.ts b/packages/jest-config/src/__tests__/parseShardPair.test.ts new file mode 100644 index 000000000000..c74b1cdd57a2 --- /dev/null +++ b/packages/jest-config/src/__tests__/parseShardPair.test.ts @@ -0,0 +1,59 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +import {parseShardPair} from '../parseShardPair'; + +it('raises an exception if shard has wrong format', () => { + expect(() => parseShardPair('mumble')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if shard pair has to many items', () => { + expect(() => parseShardPair('1/2/3')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if shard has floating points', () => { + expect(() => parseShardPair('1.0/1')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if first item in shard pair is no number', () => { + expect(() => parseShardPair('a/1')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if second item in shard pair is no number', () => { + expect(() => parseShardPair('1/a')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if shard contains negative number', () => { + expect(() => parseShardPair('1/-1')).toThrow( + 'string in the format of /', + ); +}); + +it('raises an exception if shard is zero-indexed', () => { + expect(() => parseShardPair('0/1')).toThrow( + 'requires 1-based values, received 0', + ); +}); + +it('raises an exception if shard index is larger than shard count', () => { + expect(() => parseShardPair('2/1')).toThrow( + 'requires to be lower or equal than ', + ); +}); + +it('allows valid shard format', () => { + expect(() => parseShardPair('1/2')).not.toThrow(); +}); diff --git a/packages/jest-config/src/index.ts b/packages/jest-config/src/index.ts index 1e1448bdf285..229bdf750921 100644 --- a/packages/jest-config/src/index.ts +++ b/packages/jest-config/src/index.ts @@ -154,6 +154,7 @@ const groupOptions = ( reporters: options.reporters, rootDir: options.rootDir, runTestsByPath: options.runTestsByPath, + shard: options.shard, silent: options.silent, skipFilter: options.skipFilter, snapshotFormat: options.snapshotFormat, diff --git a/packages/jest-config/src/normalize.ts b/packages/jest-config/src/normalize.ts index 36e64007f5da..6a09c510435f 100644 --- a/packages/jest-config/src/normalize.ts +++ b/packages/jest-config/src/normalize.ts @@ -34,6 +34,7 @@ import VALID_CONFIG from './ValidConfig'; import {getDisplayNameColor} from './color'; import {DEFAULT_JS_PATTERN, DEFAULT_REPORTER_LABEL} from './constants'; import getMaxWorkers from './getMaxWorkers'; +import {parseShardPair} from './parseShardPair'; import setFromArgv from './setFromArgv'; import { BULLET, @@ -1220,6 +1221,10 @@ export default async function normalize( newOptions.logHeapUsage = false; } + if (argv.shard) { + newOptions.shard = parseShardPair(argv.shard); + } + return { hasDeprecationWarnings, options: newOptions, diff --git a/packages/jest-config/src/parseShardPair.ts b/packages/jest-config/src/parseShardPair.ts new file mode 100644 index 000000000000..a9ac7c2e2af9 --- /dev/null +++ b/packages/jest-config/src/parseShardPair.ts @@ -0,0 +1,43 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +export interface ShardPair { + shardCount: number; + shardIndex: number; +} + +export const parseShardPair = (pair: string): ShardPair => { + const shardPair = pair + .split('/') + .filter(d => /^\d+$/.test(d)) + .map(d => parseInt(d, 10)) + .filter(shard => !Number.isNaN(shard)); + + const [shardIndex, shardCount] = shardPair; + + if (shardPair.length !== 2) { + throw new Error( + 'The shard option requires a string in the format of /.', + ); + } + + if (shardIndex === 0 || shardCount === 0) { + throw new Error( + 'The shard option requires 1-based values, received 0 or lower in the pair.', + ); + } + + if (shardIndex > shardCount) { + throw new Error( + 'The shard option / requires to be lower or equal than .', + ); + } + + return { + shardCount, + shardIndex, + }; +}; diff --git a/packages/jest-core/src/runJest.ts b/packages/jest-core/src/runJest.ts index 6ef4122c1ab4..7e7bf5b86501 100644 --- a/packages/jest-core/src/runJest.ts +++ b/packages/jest-core/src/runJest.ts @@ -192,6 +192,15 @@ export default async function runJest({ }), ); + if (globalConfig.shard) { + if (typeof sequencer.shard !== 'function') { + throw new Error( + `Shard ${globalConfig.shard.shardIndex}/${globalConfig.shard.shardCount} requested, but test sequencer ${Sequencer.name} in ${globalConfig.testSequencer} has no shard method.`, + ); + } + allTests = await sequencer.shard(allTests, globalConfig.shard); + } + allTests = await sequencer.sort(allTests); if (globalConfig.listTests) { diff --git a/packages/jest-test-sequencer/package.json b/packages/jest-test-sequencer/package.json index 82f8ad6c2adc..353817e074a4 100644 --- a/packages/jest-test-sequencer/package.json +++ b/packages/jest-test-sequencer/package.json @@ -20,9 +20,11 @@ "@jest/test-result": "^28.0.0-alpha.6", "graceful-fs": "^4.2.9", "jest-haste-map": "^28.0.0-alpha.6", - "jest-runtime": "^28.0.0-alpha.6" + "jest-runtime": "^28.0.0-alpha.6", + "slash": "^3.0.0" }, "devDependencies": { + "@jest/test-utils": "^28.0.0-alpha.6", "@types/graceful-fs": "^4.1.3" }, "engines": { diff --git a/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js b/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.ts similarity index 72% rename from packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js rename to packages/jest-test-sequencer/src/__tests__/test_sequencer.test.ts index f61386f11795..7d7a937ab5a9 100644 --- a/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js +++ b/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.ts @@ -6,7 +6,10 @@ */ import * as path from 'path'; -import * as fs from 'graceful-fs'; +import * as mockedFs from 'graceful-fs'; +import type {Test} from '@jest/test-result'; +import {makeProjectConfig} from '@jest/test-utils'; +import type {Context} from 'jest-runtime'; import TestSequencer from '../index'; jest.mock('graceful-fs', () => ({ @@ -17,34 +20,36 @@ jest.mock('graceful-fs', () => ({ const FAIL = 0; const SUCCESS = 1; -let sequencer; +let sequencer: TestSequencer; -const context = { - config: { +const fs = jest.mocked(mockedFs); + +const context: Context = { + config: makeProjectConfig({ cache: true, cacheDirectory: '/cache', haste: {}, name: 'test', - }, + }), hasteFS: { getSize: path => path.length, }, }; -const secondContext = { - config: { +const secondContext: Context = { + config: makeProjectConfig({ cache: true, cacheDirectory: '/cache2', haste: {}, name: 'test2', - }, + }), hasteFS: { getSize: path => path.length, }, }; -const toTests = paths => - paths.map(path => ({ +const toTests = (paths: Array) => + paths.map(path => ({ context, duration: undefined, path, @@ -125,13 +130,13 @@ test('sorts based on failures, timing information and file size', () => { ]); }); -test('writes the cache based on results without existing cache', () => { +test('writes the cache based on results without existing cache', async () => { fs.readFileSync.mockImplementationOnce(() => { throw new Error('File does not exist.'); }); const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js']; - const tests = sequencer.sort(toTests(testPaths)); + const tests = await sequencer.sort(toTests(testPaths)); sequencer.cacheResults(tests, { testResults: [ { @@ -179,7 +184,7 @@ test('returns failed tests in sorted order', () => { ]); }); -test('writes the cache based on the results', () => { +test('writes the cache based on the results', async () => { fs.readFileSync.mockImplementationOnce(() => JSON.stringify({ '/test-a.js': [SUCCESS, 5], @@ -189,7 +194,7 @@ test('writes the cache based on the results', () => { ); const testPaths = ['/test-a.js', '/test-b.js', '/test-c.js']; - const tests = sequencer.sort(toTests(testPaths)); + const tests = await sequencer.sort(toTests(testPaths)); sequencer.cacheResults(tests, { testResults: [ { @@ -223,7 +228,7 @@ test('writes the cache based on the results', () => { }); }); -test('works with multiple contexts', () => { +test('works with multiple contexts', async () => { fs.readFileSync.mockImplementationOnce(cacheName => cacheName.startsWith(`${path.sep}cache${path.sep}`) ? JSON.stringify({ @@ -240,7 +245,7 @@ test('works with multiple contexts', () => { {context, duration: null, path: '/test-b.js'}, {context: secondContext, duration: null, path: '/test-c.js'}, ]; - const tests = sequencer.sort(testPaths); + const tests = await sequencer.sort(testPaths); sequencer.cacheResults(tests, { testResults: [ { @@ -276,3 +281,82 @@ test('works with multiple contexts', () => { '/test-c.js': [SUCCESS, 3], }); }); + +test('does not shard by default', async () => { + const tests = await sequencer.shard(toTests(['/test-a.js', '/test-ab.js']), { + shardCount: 1, + shardIndex: 1, + }); + + expect(tests.map(test => test.path)).toEqual(['/test-ab.js', '/test-a.js']); +}); + +test('return first shard', async () => { + const tests = await sequencer.shard( + toTests(['/test-a.js', '/test-abc.js', '/test-ab.js']), + { + shardCount: 3, + shardIndex: 1, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-ab.js']); +}); + +test('return second shard', async () => { + const tests = await sequencer.shard( + toTests(['/test-a.js', '/test-abc.js', '/test-ab.js']), + { + shardCount: 3, + shardIndex: 2, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-abc.js']); +}); + +test('return third shard', async () => { + const tests = await sequencer.shard( + toTests(['/test-abc.js', '/test-a.js', '/test-ab.js']), + { + shardCount: 3, + shardIndex: 3, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-a.js']); +}); + +test('returns expected 100/10 shards', async () => { + const allTests = toTests(new Array(100).fill(true).map((_, i) => `/${i}.js`)); + + const shards = await Promise.all( + new Array(10).fill(true).map((_, i) => + sequencer.shard(allTests, { + shardCount: 10, + shardIndex: i + 1, + }), + ), + ); + + expect(shards.map(shard => shard.length)).toEqual([ + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + ]); +}); + +test('returns expected 100/8 shards', async () => { + const allTests = toTests(new Array(100).fill(true).map((_, i) => `/${i}.js`)); + + const shards = await Promise.all( + new Array(8).fill(true).map((_, i) => + sequencer.shard(allTests, { + shardCount: 8, + shardIndex: i + 1, + }), + ), + ); + + expect(shards.map(shard => shard.length)).toEqual([ + 13, 13, 13, 13, 13, 13, 13, 9, + ]); +}); diff --git a/packages/jest-test-sequencer/src/index.ts b/packages/jest-test-sequencer/src/index.ts index 32b0b98040e8..4b103f8a7725 100644 --- a/packages/jest-test-sequencer/src/index.ts +++ b/packages/jest-test-sequencer/src/index.ts @@ -5,7 +5,10 @@ * LICENSE file in the root directory of this source tree. */ +import * as crypto from 'crypto'; +import * as path from 'path'; import * as fs from 'graceful-fs'; +import slash = require('slash'); import type {AggregatedResult, Test} from '@jest/test-result'; import HasteMap from 'jest-haste-map'; import type {Context} from 'jest-runtime'; @@ -17,6 +20,11 @@ type Cache = { [key: string]: [0 | 1, number]; }; +export type ShardOptions = { + shardIndex: number; + shardCount: number; +}; + /** * The TestSequencer will ultimately decide which tests should run first. * It is responsible for storing and reading from a local cache @@ -66,24 +74,87 @@ export default class TestSequencer { } /** - * Sorting tests is very important because it has a great impact on the - * user-perceived responsiveness and speed of the test run. + * Select tests for shard requested via --shard=shardIndex/shardCount + * Sharding is applied before sorting * - * If such information is on cache, tests are sorted based on: - * -> Has it failed during the last run ? - * Since it's important to provide the most expected feedback as quickly - * as possible. - * -> How long it took to run ? - * Because running long tests first is an effort to minimize worker idle - * time at the end of a long test run. - * And if that information is not available they are sorted based on file size - * since big test files usually take longer to complete. + * @param tests All tests + * @param options shardIndex and shardIndex to select * - * Note that a possible improvement would be to analyse other information - * from the file other than its size. + * @example + * ```typescript + * class CustomSequencer extends Sequencer { + * shard(tests, { shardIndex, shardCount }) { + * const shardSize = Math.ceil(tests.length / options.shardCount); + * const shardStart = shardSize * (options.shardIndex - 1); + * const shardEnd = shardSize * options.shardIndex; + * return [...tests] + * .sort((a, b) => (a.path > b.path ? 1 : -1)) + * .slice(shardStart, shardEnd); + * } + * } + * ``` + */ + shard( + tests: Array, + options: ShardOptions, + ): Array | Promise> { + const shardSize = Math.ceil(tests.length / options.shardCount); + const shardStart = shardSize * (options.shardIndex - 1); + const shardEnd = shardSize * options.shardIndex; + + return tests + .map(test => { + const relativeTestPath = path.posix.relative( + slash(test.context.config.rootDir), + slash(test.path), + ); + + return { + hash: crypto + .createHash('sha1') + .update(relativeTestPath) + .digest('hex'), + test, + }; + }) + .sort((a, b) => (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0)) + .slice(shardStart, shardEnd) + .map(result => result.test); + } + + /** + * Sort test to determine order of execution + * Sorting is applied after sharding + * @param tests * + * ```typescript + * class CustomSequencer extends Sequencer { + * sort(tests) { + * const copyTests = Array.from(tests); + * return [...tests].sort((a, b) => (a.path > b.path ? 1 : -1)); + * } + * } + * ``` */ - sort(tests: Array): Array { + sort(tests: Array): Array | Promise> { + /** + * Sorting tests is very important because it has a great impact on the + * user-perceived responsiveness and speed of the test run. + * + * If such information is on cache, tests are sorted based on: + * -> Has it failed during the last run ? + * Since it's important to provide the most expected feedback as quickly + * as possible. + * -> How long it took to run ? + * Because running long tests first is an effort to minimize worker idle + * time at the end of a long test run. + * And if that information is not available they are sorted based on file size + * since big test files usually take longer to complete. + * + * Note that a possible improvement would be to analyse other information + * from the file other than its size. + * + */ const stats: {[path: string]: number} = {}; const fileSize = ({path, context: {hasteFS}}: Test) => stats[path] || (stats[path] = hasteFS.getSize(path) || 0); @@ -112,7 +183,7 @@ export default class TestSequencer { }); } - allFailedTests(tests: Array): Array { + allFailedTests(tests: Array): Array | Promise> { const hasFailed = (cache: Cache, test: Test) => cache[test.path]?.[0] === FAIL; return this.sort( diff --git a/packages/jest-test-sequencer/tsconfig.json b/packages/jest-test-sequencer/tsconfig.json index afb827643b6e..70822da7c647 100644 --- a/packages/jest-test-sequencer/tsconfig.json +++ b/packages/jest-test-sequencer/tsconfig.json @@ -9,6 +9,7 @@ "references": [ {"path": "../jest-haste-map"}, {"path": "../jest-runtime"}, - {"path": "../jest-test-result"} + {"path": "../jest-test-result"}, + {"path": "../test-utils"} ] } diff --git a/packages/jest-types/src/Config.ts b/packages/jest-types/src/Config.ts index cf0f6ae07187..80243eb45512 100644 --- a/packages/jest-types/src/Config.ts +++ b/packages/jest-types/src/Config.ts @@ -279,6 +279,11 @@ type CoverageThreshold = { global: CoverageThresholdValue; }; +type ShardConfig = { + shardIndex: number; + shardCount: number; +}; + export type GlobalConfig = { bail: number; changedSince?: string; @@ -322,6 +327,7 @@ export type GlobalConfig = { reporters?: Array; runTestsByPath: boolean; rootDir: string; + shard?: ShardConfig; silent?: boolean; skipFilter: boolean; snapshotFormat: SnapshotFormat; @@ -464,6 +470,7 @@ export type Argv = Arguments< selectProjects: Array; setupFiles: Array; setupFilesAfterEnv: Array; + shard: string; showConfig: boolean; silent: boolean; snapshotSerializers: Array; diff --git a/yarn.lock b/yarn.lock index 942cdb79e518..7ecfa4b2d8be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2853,10 +2853,12 @@ __metadata: resolution: "@jest/test-sequencer@workspace:packages/jest-test-sequencer" dependencies: "@jest/test-result": ^28.0.0-alpha.6 + "@jest/test-utils": ^28.0.0-alpha.6 "@types/graceful-fs": ^4.1.3 graceful-fs: ^4.2.9 jest-haste-map: ^28.0.0-alpha.6 jest-runtime: ^28.0.0-alpha.6 + slash: ^3.0.0 languageName: unknown linkType: soft