diff --git a/CHANGELOG.md b/CHANGELOG.md index 652452e59fef..eff6af9f3dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - `[@jest/create-cache-key-function]` Allow passing `length` argument to `createCacheKey()` function and set its default value to `16` on Windows ([#13827](https://github.com/facebook/jest/pull/13827)) - `[jest-message-util]` Add support for [AggregateError](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AggregateError) ([#13946](https://github.com/facebook/jest/pull/13946) & [#13947](https://github.com/facebook/jest/pull/13947)) - `[jest-message-util]` Add support for [Error causes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error/cause) in `test` and `it` ([#13935](https://github.com/facebook/jest/pull/13935) & [#13966](https://github.com/facebook/jest/pull/13966)) +- `[jest-reporters]` Add `summaryThreshold` option to summary reporter to allow overriding the internal threshold that is used to print the summary of all failed tests when the number of test suites surpasses it ([#13895](https://github.com/facebook/jest/pull/13895)) - `[jest-worker]` Add `start` method to worker farms ([#13937](https://github.com/facebook/jest/pull/13937)) ### Fixes diff --git a/docs/Configuration.md b/docs/Configuration.md index 342de15478b9..e935d8729985 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -1334,6 +1334,29 @@ const config: Config = { export default config; ``` +The `summary` reporter accepts options. Since it is included in the `default` reporter you may also pass the options there. + +```js tab +/** @type {import('jest').Config} */ +const config = { + reporters: [['default', {summaryThreshold: 10}]], +}; + +module.exports = config; +``` + +```ts tab +import type {Config} from 'jest'; + +const config: Config = { + reporters: [['default', {summaryThreshold: 10}]], +}; + +export default config; +``` + +The `summaryThreshold` option behaves in the following way, if the total number of test suites surpasses this threshold, a detailed summary of all failed tests will be printed after executing all the tests. It defaults to `20`. + #### Custom Reporters :::tip diff --git a/e2e/__tests__/summaryThreshold.test.ts b/e2e/__tests__/summaryThreshold.test.ts new file mode 100644 index 000000000000..5d4c9e89ccef --- /dev/null +++ b/e2e/__tests__/summaryThreshold.test.ts @@ -0,0 +1,29 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import runJest from '../runJest'; + +['default', 'summary'].forEach(reporter => { + describe(`${reporter} reporter`, () => { + test('prints failure messages when total number of test suites is over summaryThreshold', () => { + const {exitCode, stderr} = runJest('summary-threshold', [ + '--config', + JSON.stringify({ + reporters: [[reporter, {summaryThreshold: 2}]], + }), + ]); + + expect(exitCode).toBe(1); + expect(stderr).toMatch( + /Summary of all failing tests(\n|.)*expect\(1\)\.toBe\(0\)/, + ); + expect(stderr).toMatch( + /Summary of all failing tests(\n|.)*expect\(2\)\.toBe\(0\)/, + ); + }); + }); +}); diff --git a/e2e/summary-threshold/__tests__/summarySuite1.test.js b/e2e/summary-threshold/__tests__/summarySuite1.test.js new file mode 100644 index 000000000000..87dda7f776f4 --- /dev/null +++ b/e2e/summary-threshold/__tests__/summarySuite1.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('fails', () => { + expect(1).toBe(0); +}); diff --git a/e2e/summary-threshold/__tests__/summarySuite2.test.js b/e2e/summary-threshold/__tests__/summarySuite2.test.js new file mode 100644 index 000000000000..ea74e07cd926 --- /dev/null +++ b/e2e/summary-threshold/__tests__/summarySuite2.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('fails', () => { + expect(2).toBe(0); +}); diff --git a/e2e/summary-threshold/__tests__/summarySuite3.test.js b/e2e/summary-threshold/__tests__/summarySuite3.test.js new file mode 100644 index 000000000000..ba43fbad6561 --- /dev/null +++ b/e2e/summary-threshold/__tests__/summarySuite3.test.js @@ -0,0 +1,11 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +test('passes', () => { + expect(1).toBe(1); +}); diff --git a/e2e/summary-threshold/package.json b/e2e/summary-threshold/package.json new file mode 100644 index 000000000000..148788b25446 --- /dev/null +++ b/e2e/summary-threshold/package.json @@ -0,0 +1,5 @@ +{ + "jest": { + "testEnvironment": "node" + } +} diff --git a/packages/jest-core/src/TestScheduler.ts b/packages/jest-core/src/TestScheduler.ts index 708f5ae86951..00ad2b053768 100644 --- a/packages/jest-core/src/TestScheduler.ts +++ b/packages/jest-core/src/TestScheduler.ts @@ -17,6 +17,7 @@ import { Reporter, ReporterContext, SummaryReporter, + SummaryReporterOptions, VerboseReporter, } from '@jest/reporters'; import { @@ -336,12 +337,12 @@ class TestScheduler { async _setupReporters() { const {collectCoverage: coverage, notify, verbose} = this._globalConfig; const reporters = this._globalConfig.reporters || [['default', {}]]; - let summary = false; + let summaryOptions: SummaryReporterOptions | null = null; for (const [reporter, options] of reporters) { switch (reporter) { case 'default': - summary = true; + summaryOptions = options; verbose ? this.addReporter(new VerboseReporter(this._globalConfig)) : this.addReporter(new DefaultReporter(this._globalConfig)); @@ -353,7 +354,7 @@ class TestScheduler { ); break; case 'summary': - summary = true; + summaryOptions = options; break; default: await this._addCustomReporter(reporter, options); @@ -368,8 +369,8 @@ class TestScheduler { this.addReporter(new CoverageReporter(this._globalConfig, this._context)); } - if (summary) { - this.addReporter(new SummaryReporter(this._globalConfig)); + if (summaryOptions != null) { + this.addReporter(new SummaryReporter(this._globalConfig, summaryOptions)); } } diff --git a/packages/jest-reporters/src/SummaryReporter.ts b/packages/jest-reporters/src/SummaryReporter.ts index 90e8095b4d2f..ed7842fedebb 100644 --- a/packages/jest-reporters/src/SummaryReporter.ts +++ b/packages/jest-reporters/src/SummaryReporter.ts @@ -19,8 +19,6 @@ import getSnapshotSummary from './getSnapshotSummary'; import getSummary from './getSummary'; import type {ReporterOnStartOptions} from './types'; -const TEST_SUMMARY_THRESHOLD = 20; - const NPM_EVENTS = new Set([ 'prepublish', 'publish', @@ -51,16 +49,35 @@ const NPM_EVENTS = new Set([ const {npm_config_user_agent, npm_lifecycle_event, npm_lifecycle_script} = process.env; +export type SummaryReporterOptions = { + summaryThreshold?: number; +}; + export default class SummaryReporter extends BaseReporter { private _estimatedTime: number; private readonly _globalConfig: Config.GlobalConfig; + private readonly _summaryThreshold: number; static readonly filename = __filename; - constructor(globalConfig: Config.GlobalConfig) { + constructor( + globalConfig: Config.GlobalConfig, + options?: SummaryReporterOptions, + ) { super(); this._globalConfig = globalConfig; this._estimatedTime = 0; + this._validateOptions(options); + this._summaryThreshold = options?.summaryThreshold ?? 20; + } + + private _validateOptions(options?: SummaryReporterOptions) { + if ( + options?.summaryThreshold && + typeof options.summaryThreshold !== 'number' + ) { + throw new TypeError('The option summaryThreshold should be a number'); + } } // If we write more than one character at a time it is possible that @@ -176,7 +193,7 @@ export default class SummaryReporter extends BaseReporter { const runtimeErrors = aggregatedResults.numRuntimeErrorTestSuites; if ( failedTests + runtimeErrors > 0 && - aggregatedResults.numTotalTestSuites > TEST_SUMMARY_THRESHOLD + aggregatedResults.numTotalTestSuites > this._summaryThreshold ) { this.log(chalk.bold('Summary of all failing tests')); aggregatedResults.testResults.forEach(testResult => { diff --git a/packages/jest-reporters/src/__tests__/SummaryReporter.test.js b/packages/jest-reporters/src/__tests__/SummaryReporter.test.js index 159cb78d83f2..fb83c448194a 100644 --- a/packages/jest-reporters/src/__tests__/SummaryReporter.test.js +++ b/packages/jest-reporters/src/__tests__/SummaryReporter.test.js @@ -160,3 +160,78 @@ test('snapshots all have results (after update)', () => { testReporter.onRunComplete(new Set(), aggregatedResults); expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot(); }); + +describe('summaryThreshold option', () => { + const aggregatedResults = { + numFailedTestSuites: 1, + numFailedTests: 1, + numPassedTestSuites: 2, + numRuntimeErrorTestSuites: 0, + numTotalTestSuites: 3, + numTotalTests: 3, + snapshot: { + filesRemovedList: [], + filesUnmatched: 0, + total: 0, + uncheckedKeysByFile: [], + unmatched: 0, + }, + startTime: 0, + testResults: [ + { + failureMessage: 'FailureMessage1', + numFailingTests: 1, + testFilePath: 'path1', + }, + { + failureMessage: 'FailureMessage2', + numFailingTests: 1, + testFilePath: 'path2', + }, + ], + }; + + it('Should print failure messages when number of test suites is over the threshold', () => { + const options = { + summaryThreshold: aggregatedResults.numTotalTestSuites - 1, + }; + + requireReporter(); + const testReporter = new SummaryReporter(globalConfig, options); + testReporter.onRunComplete(new Set(), aggregatedResults); + expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot(); + }); + + it('Should not print failure messages when number of test suites is under the threshold', () => { + const options = { + summaryThreshold: aggregatedResults.numTotalTestSuites + 1, + }; + + requireReporter(); + const testReporter = new SummaryReporter(globalConfig, options); + testReporter.onRunComplete(new Set(), aggregatedResults); + expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot(); + }); + + it('Should not print failure messages when number of test suites is equal to the threshold', () => { + const options = { + summaryThreshold: aggregatedResults.numTotalTestSuites, + }; + + requireReporter(); + const testReporter = new SummaryReporter(globalConfig, options); + testReporter.onRunComplete(new Set(), aggregatedResults); + expect(results.join('').replace(/\\/g, '/')).toMatchSnapshot(); + }); + + it('Should throw error if threshold is not a number', () => { + const options = { + summaryThreshold: 'not a number', + }; + + requireReporter(); + expect(() => new SummaryReporter(globalConfig, options)).toThrow( + 'The option summaryThreshold should be a number', + ); + }); +}); diff --git a/packages/jest-reporters/src/__tests__/__snapshots__/SummaryReporter.test.js.snap b/packages/jest-reporters/src/__tests__/__snapshots__/SummaryReporter.test.js.snap index 42e5e5366851..60b44ee0f551 100644 --- a/packages/jest-reporters/src/__tests__/__snapshots__/SummaryReporter.test.js.snap +++ b/packages/jest-reporters/src/__tests__/__snapshots__/SummaryReporter.test.js.snap @@ -59,3 +59,36 @@ exports[`snapshots needs update with yarn test 1`] = ` Ran all test suites. " `; + +exports[`summaryThreshold option Should not print failure messages when number of test suites is equal to the threshold 1`] = ` +"Test Suites: 1 failed, 2 passed, 3 total +Tests: 1 failed, 3 total +Snapshots: 0 total +Time: 0.01 s +Ran all test suites. +" +`; + +exports[`summaryThreshold option Should not print failure messages when number of test suites is under the threshold 1`] = ` +"Test Suites: 1 failed, 2 passed, 3 total +Tests: 1 failed, 3 total +Snapshots: 0 total +Time: 0.01 s +Ran all test suites. +" +`; + +exports[`summaryThreshold option Should print failure messages when number of test suites is over the threshold 1`] = ` +"Summary of all failing tests + FAIL ../path1 +FailureMessage1 + FAIL ../path2 +FailureMessage2 + +Test Suites: 1 failed, 2 passed, 3 total +Tests: 1 failed, 3 total +Snapshots: 0 total +Time: 0.01 s +Ran all test suites. +" +`; diff --git a/packages/jest-reporters/src/index.ts b/packages/jest-reporters/src/index.ts index 37a75b11db2c..5e39dbdd477f 100644 --- a/packages/jest-reporters/src/index.ts +++ b/packages/jest-reporters/src/index.ts @@ -30,6 +30,7 @@ export {default as GitHubActionsReporter} from './GitHubActionsReporter'; export {default as NotifyReporter} from './NotifyReporter'; export {default as SummaryReporter} from './SummaryReporter'; export {default as VerboseReporter} from './VerboseReporter'; +export type {SummaryReporterOptions} from './SummaryReporter'; export type { Reporter, ReporterOnStartOptions,