From 3052d3b45618f3d126e30751d52b81c165f60e91 Mon Sep 17 00:00:00 2001 From: Mario Nebl Date: Thu, 3 Mar 2022 22:19:33 +1100 Subject: [PATCH] feat: implement --shard option #6270 --- .github/workflows/nodejs.yml | 78 +++++++++++++-- docs/CLI.md | 10 ++ .../jest-cli/src/__tests__/cli/args.test.ts | 98 +++++++++++++------ packages/jest-cli/src/cli/args.ts | 34 +++++++ .../src/__tests__/normalize.test.ts | 10 ++ packages/jest-config/src/index.ts | 1 + packages/jest-config/src/normalize.ts | 13 +++ packages/jest-core/src/runJest.ts | 1 + .../src/__tests__/test_sequencer.test.js | 71 ++++++++++++++ packages/jest-test-sequencer/src/index.ts | 22 +++++ packages/jest-types/src/Config.ts | 7 ++ 11 files changed, 308 insertions(+), 37 deletions(-) diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 50871be0f1ea..5e9a35df6ec5 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -65,8 +65,42 @@ jobs: run: yarn lint:prettier:ci - name: check copyright headers run: yarn check-copyright-headers - test: - name: Node v${{ matrix.node-version }} on ${{ matrix.os }} + + test-1: + name: Node v${{ matrix.node-version }} on ${{ matrix.os }} (1/2) + strategy: + fail-fast: false + matrix: + node-version: [12.x, 14.x, 16.x, 17.x] + os: [ubuntu-latest, macOS-latest, windows-latest] + runs-on: ${{ matrix.os }} + needs: prepare-yarn-cache + + steps: + - name: Set git config + shell: bash + run: | + git config --global core.autocrlf false + git config --global core.symlinks true + if: runner.os == 'Windows' + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + cache: yarn + - name: install + run: yarn --immutable + - name: build + run: yarn build:js + - name: Get number of CPU cores + id: cpu-cores + uses: SimenB/github-actions-cpu-cores@v1 + - name: run tests + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=1/2 + + test-2: + name: Node v${{ matrix.node-version }} on ${{ matrix.os }} (2/2) strategy: fail-fast: false matrix: @@ -96,10 +130,42 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests - run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=2/2 + + test-jasmine-1: + name: Node LTS on ${{ matrix.os }} using jest-jasmine2 (1/2) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + runs-on: ${{ matrix.os }} + needs: prepare-yarn-cache + + steps: + - name: Set git config + shell: bash + run: | + git config --global core.autocrlf false + git config --global core.symlinks true + if: runner.os == 'Windows' + - uses: actions/checkout@v3 + - name: Use Node.js LTS + uses: actions/setup-node@v3 + with: + node-version: lts/* + cache: yarn + - name: install + run: yarn --immutable + - name: build + run: yarn build:js + - name: Get number of CPU cores + id: cpu-cores + uses: SimenB/github-actions-cpu-cores@v1 + - name: run tests using jest-jasmine + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=1/2 - test-jasmine: - name: Node LTS on ${{ matrix.os }} using jest-jasmine2 + test-jasmine-2: + name: Node LTS on ${{ matrix.os }} using jest-jasmine2 (2/2) strategy: fail-fast: false matrix: @@ -128,7 +194,7 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine - run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=2/2 test-coverage: name: Node LTS on Ubuntu with coverage diff --git a/docs/CLI.md b/docs/CLI.md index d465b78d34b3..37e7df52e37c 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -338,6 +338,16 @@ Run only the tests of the specified projects. Jest uses the attribute `displayNa A list of paths to modules that run some code to configure or to set up the testing framework before each test. Beware that files imported by the setup scripts will not be mocked during testing. +### `--shard` + +Shard suite to execute in on multiple machines. For example, to split the suite into three shards, each running one third of the tests: + +``` +jest --shard=1/3 +jest --shard=2/3 +jest --shard=3/3 +``` + ### `--showConfig` Print your Jest config and then exits. diff --git a/packages/jest-cli/src/__tests__/cli/args.test.ts b/packages/jest-cli/src/__tests__/cli/args.test.ts index ec302f70d734..7fafeeed7bf1 100644 --- a/packages/jest-cli/src/__tests__/cli/args.test.ts +++ b/packages/jest-cli/src/__tests__/cli/args.test.ts @@ -11,84 +11,78 @@ import {constants} from 'jest-config'; import {buildArgv} from '../../cli'; import {check} from '../../cli/args'; +const argv = (input: Partial): Config.Argv => input as Config.Argv; + describe('check', () => { it('returns true if the arguments are valid', () => { - const argv = {} as Config.Argv; - expect(check(argv)).toBe(true); + expect(check(argv({}))).toBe(true); }); it('raises an exception if runInBand and maxWorkers are both specified', () => { - const argv = {maxWorkers: 2, runInBand: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({maxWorkers: 2, runInBand: true}))).toThrow( 'Both --runInBand and --maxWorkers were specified', ); }); it('raises an exception if onlyChanged and watchAll are both specified', () => { - const argv = {onlyChanged: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({onlyChanged: true, watchAll: true}))).toThrow( 'Both --onlyChanged and --watchAll were specified', ); }); it('raises an exception if onlyFailures and watchAll are both specified', () => { - const argv = {onlyFailures: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({onlyFailures: true, watchAll: true}))).toThrow( 'Both --onlyFailures and --watchAll were specified', ); }); it('raises an exception when lastCommit and watchAll are both specified', () => { - const argv = {lastCommit: true, watchAll: true} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({lastCommit: true, watchAll: true}))).toThrow( 'Both --lastCommit and --watchAll were specified', ); }); it('raises an exception if findRelatedTests is specified with no file paths', () => { - const argv = { - _: [] as Array, - findRelatedTests: true, - } as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => + check( + argv({ + _: [], + findRelatedTests: true, + }), + ), + ).toThrow( 'The --findRelatedTests option requires file paths to be specified', ); }); it('raises an exception if maxWorkers is specified with no number', () => { - const argv = {maxWorkers: undefined} as unknown as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({maxWorkers: undefined}))).toThrow( 'The --maxWorkers (-w) option requires a number or string to be specified', ); }); it('allows maxWorkers to be a %', () => { - const argv = {maxWorkers: '50%'} as unknown as Config.Argv; - expect(() => check(argv)).not.toThrow(); + expect(() => check(argv({maxWorkers: '50%'}))).not.toThrow(); }); test.each(constants.JEST_CONFIG_EXT_ORDER.map(e => e.substring(1)))( 'allows using "%s" file for --config option', ext => { + expect(() => check(argv({config: `jest.config.${ext}`}))).not.toThrow(); expect(() => - check({config: `jest.config.${ext}`} as Config.Argv), - ).not.toThrow(); - expect(() => - check({config: `../test/test/my_conf.${ext}`} as Config.Argv), + check(argv({config: `../test/test/my_conf.${ext}`})), ).not.toThrow(); }, ); it('raises an exception if selectProjects is not provided any project names', () => { - const argv: Config.Argv = {selectProjects: []} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({selectProjects: []}))).toThrow( 'The --selectProjects option requires the name of at least one project to be specified.\n', ); }); it('raises an exception if config is not a valid JSON string', () => { - const argv = {config: 'x:1'} as Config.Argv; - expect(() => check(argv)).toThrow( + expect(() => check(argv({config: 'x:1'}))).toThrow( 'The --config option requires a JSON string literal, or a file path with one of these extensions: .js, .ts, .mjs, .cjs, .json', ); }); @@ -97,13 +91,55 @@ describe('check', () => { const message = 'The --config option requires a JSON string literal, or a file path with one of these extensions: .js, .ts, .mjs, .cjs, .json'; - expect(() => check({config: 'jest.configjs'} as Config.Argv)).toThrow( - message, + expect(() => check(argv({config: 'jest.configjs'}))).toThrow(message); + expect(() => check(argv({config: 'jest.config.exe'}))).toThrow(message); + }); + + it('raises an exception if shard has wrong format', () => { + expect(() => check(argv({shard: 'mumblemuble'}))).toThrow( + /string in the format of \//, + ); + }); + + it('raises an exception if shard pair has to many items', () => { + expect(() => check(argv({shard: '1/2/2'}))).toThrow( + /string in the format of \//, + ); + }); + + it('raises an exception if shard has floating points', () => { + expect(() => check(argv({shard: '1.0/1'}))).toThrow( + /string in the format of \//, + ); + }); + + it('raises an exception if first item in shard pair is no number', () => { + expect(() => check(argv({shard: 'a/1'}))).toThrow( + /string in the format of \//, + ); + }); + + it('raises an exception if second item in shard pair is no number', () => { + expect(() => check(argv({shard: '1/a'}))).toThrow( + /string in the format of \//, + ); + }); + + it('raises an exception if shard is zero-indexed', () => { + expect(() => check(argv({shard: '0/1'}))).toThrow( + /requires 1-based values, received 0/, ); - expect(() => check({config: 'jest.config.exe'} as Config.Argv)).toThrow( - message, + }); + + it('raises an exception if shard index is larger than shard count', () => { + expect(() => check(argv({shard: '2/1'}))).toThrow( + /requires to be lower or equal than /, ); }); + + it('allows valid shard format', () => { + expect(() => check(argv({shard: '1/2'}))).not.toThrow(); + }); }); describe('buildArgv', () => { diff --git a/packages/jest-cli/src/cli/args.ts b/packages/jest-cli/src/cli/args.ts index 92c13247f646..1d2f140404d5 100644 --- a/packages/jest-cli/src/cli/args.ts +++ b/packages/jest-cli/src/cli/args.ts @@ -87,6 +87,34 @@ export function check(argv: Config.Argv): true { ); } + if (argv.shard) { + const shardPair = argv?.shard + .split('/') + .filter(d => /^\d+$/.test(d)) + .map(d => parseInt(d, 10)) + .filter((shard: number) => !Number.isNaN(shard)); + + if (shardPair.length !== 2) { + throw new Error( + 'The --shard option requires a string in the format of /.\n Example usage jest --shard=1/5', + ); + } + + const [shardIndex, shardCount] = shardPair; + + if (shardIndex === 0 || shardCount === 0) { + throw new Error( + 'The --shard option requires 1-based values, received 0 in the pair.\n Example usage jest --shard=1/5', + ); + } + + if (shardIndex > shardCount) { + throw new Error( + 'The --shard option / requires to be lower or equal than .\n Example usage jest --shard=1/5', + ); + } + } + return true; } @@ -521,6 +549,12 @@ export const options = { string: true, type: 'array', }, + shard: { + description: + 'Shard tests and execute only the selected shard, specify in ' + + 'the form "current/all". 1-based, for example "3/5"', + type: 'string', + }, showConfig: { description: 'Print your jest config and then exits.', type: 'boolean', diff --git a/packages/jest-config/src/__tests__/normalize.test.ts b/packages/jest-config/src/__tests__/normalize.test.ts index 7c96711a8c89..ccca146c1c54 100644 --- a/packages/jest-config/src/__tests__/normalize.test.ts +++ b/packages/jest-config/src/__tests__/normalize.test.ts @@ -1972,3 +1972,13 @@ describe('moduleLoader', () => { expect(console.warn).toMatchSnapshot(); }); }); + +describe('shards', () => { + it('should be object if defined', async () => { + const {options} = await normalize({rootDir: '/root/'}, { + shard: '1/2', + } as Config.Argv); + + expect(options.shard).toEqual({shardCount: 2, shardIndex: 1}); + }); +}); diff --git a/packages/jest-config/src/index.ts b/packages/jest-config/src/index.ts index 1e1448bdf285..229bdf750921 100644 --- a/packages/jest-config/src/index.ts +++ b/packages/jest-config/src/index.ts @@ -154,6 +154,7 @@ const groupOptions = ( reporters: options.reporters, rootDir: options.rootDir, runTestsByPath: options.runTestsByPath, + shard: options.shard, silent: options.silent, skipFilter: options.skipFilter, snapshotFormat: options.snapshotFormat, diff --git a/packages/jest-config/src/normalize.ts b/packages/jest-config/src/normalize.ts index 36e64007f5da..3930dc1b4164 100644 --- a/packages/jest-config/src/normalize.ts +++ b/packages/jest-config/src/normalize.ts @@ -1220,6 +1220,19 @@ export default async function normalize( newOptions.logHeapUsage = false; } + if (argv.shard) { + const [shardIndex, shardCount] = argv?.shard + .split('/') + .filter(d => /^\d+$/.test(d)) + .map(d => parseInt(d, 10)) + .filter((shard: number) => !Number.isNaN(shard)); + + newOptions.shard = { + shardCount, + shardIndex, + }; + } + return { hasDeprecationWarnings, options: newOptions, diff --git a/packages/jest-core/src/runJest.ts b/packages/jest-core/src/runJest.ts index 6ef4122c1ab4..d087dc2c565f 100644 --- a/packages/jest-core/src/runJest.ts +++ b/packages/jest-core/src/runJest.ts @@ -192,6 +192,7 @@ export default async function runJest({ }), ); + allTests = sequencer.shard?.(allTests, globalConfig.shard) ?? allTests; allTests = await sequencer.sort(allTests); if (globalConfig.listTests) { diff --git a/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js b/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js index f61386f11795..c5000067d106 100644 --- a/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js +++ b/packages/jest-test-sequencer/src/__tests__/test_sequencer.test.js @@ -276,3 +276,74 @@ test('works with multiple contexts', () => { '/test-c.js': [SUCCESS, 3], }); }); + +test('does not shard by default', () => { + const tests = sequencer.shard(toTests(['/test-a.js', '/test-ab.js'])); + expect(tests.map(test => test.path)).toEqual(['/test-a.js', '/test-ab.js']); +}); + +test('return first shard', () => { + const tests = sequencer.shard( + toTests(['/test-a.js', '/test-abc.js', '/test-ab.js']), + { + shardCount: 2, + shardIndex: 1, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-a.js', '/test-ab.js']); +}); + +test('return second shard', () => { + const tests = sequencer.shard( + toTests(['/test-ab.js', '/test-abc.js', '/test-a.js']), + { + shardCount: 2, + shardIndex: 2, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-abc.js']); +}); + +test('return third shard', () => { + const tests = sequencer.shard( + toTests(['/test-abc.js', '/test-a.js', '/test-ab.js']), + { + shardCount: 3, + shardIndex: 3, + }, + ); + + expect(tests.map(test => test.path)).toEqual(['/test-abc.js']); +}); + +test('returns expected 100/10 shards', () => { + const allTests = new Array(100).fill(true).map((_, i) => `/${i}.js`); + + const shards = new Array(10).fill(true).map((_, i) => + sequencer.shard(allTests, { + shardCount: 10, + shardIndex: i + 1, + }), + ); + + expect(shards.map(shard => shard.length)).toEqual([ + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + ]); +}); + +test('returns expected 100/8 shards', () => { + const allTests = new Array(100).fill(true).map((_, i) => `/${i}.js`); + + const shards = new Array(8).fill(true).map((_, i) => + sequencer.shard(allTests, { + shardCount: 8, + shardIndex: i + 1, + }), + ); + + expect(shards.map(shard => shard.length)).toEqual([ + 13, 13, 13, 13, 13, 13, 13, 9, + ]); +}); diff --git a/packages/jest-test-sequencer/src/index.ts b/packages/jest-test-sequencer/src/index.ts index 32b0b98040e8..b3f2c7ed5da3 100644 --- a/packages/jest-test-sequencer/src/index.ts +++ b/packages/jest-test-sequencer/src/index.ts @@ -17,6 +17,11 @@ type Cache = { [key: string]: [0 | 1, number]; }; +export type SortOptions = { + shardIndex: number; + shardCount: number; +}; + /** * The TestSequencer will ultimately decide which tests should run first. * It is responsible for storing and reading from a local cache @@ -65,6 +70,23 @@ export default class TestSequencer { return cache; } + shard( + tests: Array, + options: SortOptions = {shardCount: 1, shardIndex: 1}, + ): Array { + if (options.shardCount > 1) { + const shardSize = Math.ceil(tests.length / options.shardCount); + const shardStart = shardSize * (options.shardIndex - 1); + const shardEnd = shardSize * options.shardIndex; + + return [...tests] + .sort((a, b) => (a.path > b.path ? 1 : -1)) + .slice(shardStart, shardEnd); + } + + return tests; + } + /** * Sorting tests is very important because it has a great impact on the * user-perceived responsiveness and speed of the test run. diff --git a/packages/jest-types/src/Config.ts b/packages/jest-types/src/Config.ts index cf0f6ae07187..80243eb45512 100644 --- a/packages/jest-types/src/Config.ts +++ b/packages/jest-types/src/Config.ts @@ -279,6 +279,11 @@ type CoverageThreshold = { global: CoverageThresholdValue; }; +type ShardConfig = { + shardIndex: number; + shardCount: number; +}; + export type GlobalConfig = { bail: number; changedSince?: string; @@ -322,6 +327,7 @@ export type GlobalConfig = { reporters?: Array; runTestsByPath: boolean; rootDir: string; + shard?: ShardConfig; silent?: boolean; skipFilter: boolean; snapshotFormat: SnapshotFormat; @@ -464,6 +470,7 @@ export type Argv = Arguments< selectProjects: Array; setupFiles: Array; setupFilesAfterEnv: Array; + shard: string; showConfig: boolean; silent: boolean; snapshotSerializers: Array;