From 918636142791a3dd4ddfe9367149a90437bd6da9 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Sat, 1 Aug 2020 19:30:33 +1200 Subject: [PATCH] Add `failureDetails` property to circus test results (#9496) --- CHANGELOG.md | 2 + e2e/__tests__/failureDetailsProperty.test.ts | 217 ++++++++++++++++++ .../__tests__/tests.test.js | 37 +++ e2e/failureDetails-property/myreporter.js | 22 ++ e2e/failureDetails-property/package.json | 8 + .../jestAdapterInit.ts | 1 + packages/jest-circus/src/utils.ts | 25 +- packages/jest-jasmine2/src/reporter.ts | 2 + packages/jest-types/src/Circus.ts | 8 + packages/jest-types/src/TestResult.ts | 1 + 10 files changed, 311 insertions(+), 12 deletions(-) create mode 100644 e2e/__tests__/failureDetailsProperty.test.ts create mode 100644 e2e/failureDetails-property/__tests__/tests.test.js create mode 100644 e2e/failureDetails-property/myreporter.js create mode 100644 e2e/failureDetails-property/package.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8fa2297afe..2e87445cb0e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Features +- `[jest-circus, jest-jasmine2]` Include `failureDetails` property in test results ([#9496](https://github.com/facebook/jest/pull/9496)) + ### Fixes ### Chore & Maintenance diff --git a/e2e/__tests__/failureDetailsProperty.test.ts b/e2e/__tests__/failureDetailsProperty.test.ts new file mode 100644 index 000000000000..3e4130a0052b --- /dev/null +++ b/e2e/__tests__/failureDetailsProperty.test.ts @@ -0,0 +1,217 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {isJestCircusRun} from '@jest/test-utils'; +import runJest from '../runJest'; + +const removeStackTraces = (stdout: string) => + stdout.replace( + /at (new Promise \(\)|.+:\d+:\d+\)?)/g, + 'at ', + ); + +test('that the failureDetails property is set', () => { + const {stdout, stderr} = runJest('failureDetails-property', [ + 'tests.test.js', + ]); + + // safety check: if the reporter errors it'll show up here + expect(stderr).toStrictEqual(''); + + const output = JSON.parse(removeStackTraces(stdout)); + + if (isJestCircusRun()) { + expect(output).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "matcherResult": Object { + "actual": true, + "expected": false, + "name": "toBe", + "pass": false, + }, + }, + ], + Array [ + Object { + "matcherResult": Object { + "actual": true, + "expected": false, + "name": "toBe", + "pass": false, + }, + }, + ], + Array [ + Object { + "matcherResult": Object { + "actual": "Object { + \\"p1\\": \\"hello\\", + \\"p2\\": \\"world\\", + }", + "expected": "Object { + \\"p1\\": \\"hello\\", + \\"p2\\": \\"sunshine\\", + }", + "name": "toMatchInlineSnapshot", + "pass": false, + }, + }, + ], + Array [ + Object {}, + ], + Array [ + Object { + "message": "expect(received).rejects.toThrowError() + + Received promise resolved instead of rejected + Resolved to value: 1", + }, + ], + ] + `); + } else { + expect(output).toMatchInlineSnapshot(` + Array [ + Array [ + Object { + "actual": "", + "error": Object { + "matcherResult": Object { + "actual": true, + "expected": false, + "name": "toBe", + "pass": false, + }, + }, + "expected": "", + "matcherName": "", + "message": "Error: expect(received).toBe(expected) // Object.is equality + + Expected: false + Received: true", + "passed": false, + "stack": "Error: expect(received).toBe(expected) // Object.is equality + + Expected: false + Received: true + at ", + }, + ], + Array [ + Object { + "actual": "", + "error": Object { + "matcherResult": Object { + "actual": true, + "expected": false, + "name": "toBe", + "pass": false, + }, + }, + "expected": "", + "matcherName": "", + "message": "Error: expect(received).toBe(expected) // Object.is equality + + Expected: false + Received: true", + "passed": false, + "stack": "Error: expect(received).toBe(expected) // Object.is equality + + Expected: false + Received: true + at ", + }, + ], + Array [ + Object { + "actual": "", + "error": Object { + "matcherResult": Object { + "actual": "Object { + \\"p1\\": \\"hello\\", + \\"p2\\": \\"world\\", + }", + "expected": "Object { + \\"p1\\": \\"hello\\", + \\"p2\\": \\"sunshine\\", + }", + "name": "toMatchInlineSnapshot", + "pass": false, + }, + }, + "expected": "", + "matcherName": "", + "message": "expect(received).toMatchInlineSnapshot(snapshot) + + Snapshot name: \`my test a snapshot failure 1\` + + - Snapshot - 1 + + Received + 1 + + Object { + \\"p1\\": \\"hello\\", + - \\"p2\\": \\"sunshine\\", + + \\"p2\\": \\"world\\", + }", + "passed": false, + "stack": "Error: expect(received).toMatchInlineSnapshot(snapshot) + + Snapshot name: \`my test a snapshot failure 1\` + + - Snapshot - 1 + + Received + 1 + + Object { + \\"p1\\": \\"hello\\", + - \\"p2\\": \\"sunshine\\", + + \\"p2\\": \\"world\\", + } + at ", + }, + ], + Array [ + Object { + "actual": "", + "error": Object {}, + "expected": "", + "matcherName": "", + "message": "Error", + "passed": false, + "stack": "Error: + at ", + }, + ], + Array [ + Object { + "actual": "", + "error": Object { + "message": "expect(received).rejects.toThrowError() + + Received promise resolved instead of rejected + Resolved to value: 1", + }, + "expected": "", + "matcherName": "", + "message": "Error: expect(received).rejects.toThrowError() + + Received promise resolved instead of rejected + Resolved to value: 1", + "passed": false, + "stack": "Error: expect(received).rejects.toThrowError() + + Received promise resolved instead of rejected + Resolved to value: 1 + at ", + }, + ], + ] + `); + } +}); diff --git a/e2e/failureDetails-property/__tests__/tests.test.js b/e2e/failureDetails-property/__tests__/tests.test.js new file mode 100644 index 000000000000..e9134e93a0de --- /dev/null +++ b/e2e/failureDetails-property/__tests__/tests.test.js @@ -0,0 +1,37 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +'use strict'; + +describe('my test', () => { + test('it passes', () => { + expect(true).toBe(false); + }); + + it('fails :(', () => { + expect(true).toBe(false); + }); + + test('a snapshot failure', () => { + expect({ + p1: 'hello', + p2: 'world', + }).toMatchInlineSnapshot(` + Object { + "p1": "hello", + "p2": "sunshine", + } + `); + }); +}); + +it('throws!', () => { + throw new Error(); +}); + +test('promise rejection', async () => { + await expect(Promise.resolve(1)).rejects.toThrowError(); +}); diff --git a/e2e/failureDetails-property/myreporter.js b/e2e/failureDetails-property/myreporter.js new file mode 100644 index 000000000000..7abe771257fd --- /dev/null +++ b/e2e/failureDetails-property/myreporter.js @@ -0,0 +1,22 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +'use strict'; + +class MyCustomReporter { + onRunComplete(contexts, results) { + console.log( + JSON.stringify( + results.testResults[0].testResults.map(f => f.failureDetails), + null, + 2, + ), + ); + } +} + +module.exports = MyCustomReporter; diff --git a/e2e/failureDetails-property/package.json b/e2e/failureDetails-property/package.json new file mode 100644 index 000000000000..fcee5832dcbe --- /dev/null +++ b/e2e/failureDetails-property/package.json @@ -0,0 +1,8 @@ +{ + "jest": { + "testEnvironment": "node", + "reporters": [ + "/myreporter.js" + ] + } +} diff --git a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts index de8850dbee85..fba12da73473 100644 --- a/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts +++ b/packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts @@ -193,6 +193,7 @@ export const runAndTransformResultsToJestFormat = async ({ return { ancestorTitles, duration: testResult.duration, + failureDetails: testResult.errorsDetailed, failureMessages: testResult.errors, fullName: title ? ancestorTitles.concat(title).join(' ') diff --git a/packages/jest-circus/src/utils.ts b/packages/jest-circus/src/utils.ts index acf8c56579ec..82b4b78e52d4 100644 --- a/packages/jest-circus/src/utils.ts +++ b/packages/jest-circus/src/utils.ts @@ -280,7 +280,7 @@ export const makeRunResult = ( unhandledErrors: Array, ): Circus.RunResult => ({ testResults: makeTestResults(describeBlock), - unhandledErrors: unhandledErrors.map(_formatError), + unhandledErrors: unhandledErrors.map(_getError).map(getErrorStack), }); export const makeSingleTestResult = ( @@ -313,9 +313,12 @@ export const makeSingleTestResult = ( } } + const errorsDetailed = test.errors.map(_getError); + return { duration: test.duration, - errors: test.errors.map(_formatError), + errors: errorsDetailed.map(getErrorStack), + errorsDetailed, invocations: test.invocations, location, status, @@ -357,9 +360,9 @@ export const getTestID = (test: Circus.TestEntry): string => { return titles.join(' '); }; -const _formatError = ( +const _getError = ( errors?: Circus.Exception | [Circus.Exception | undefined, Circus.Exception], -): string => { +): Error => { let error; let asyncError; @@ -371,20 +374,17 @@ const _formatError = ( asyncError = new Error(); } - if (error) { - if (error.stack) { - return error.stack; - } - if (error.message) { - return error.message; - } + if (error && (error.stack || error.message)) { + return error; } asyncError.message = `thrown: ${prettyFormat(error, {maxDepth: 3})}`; - return asyncError.stack; + return asyncError; }; +const getErrorStack = (error: Error): string => error.stack || error.message; + export const addErrorToEachTestUnderDescribe = ( describeBlock: Circus.DescribeBlock, error: Circus.Exception, @@ -433,6 +433,7 @@ export const parseSingleTestResult = ( return { ancestorTitles, duration: testResult.duration, + failureDetails: testResult.errorsDetailed, failureMessages: Array.from(testResult.errors), fullName: title ? ancestorTitles.concat(title).join(' ') diff --git a/packages/jest-jasmine2/src/reporter.ts b/packages/jest-jasmine2/src/reporter.ts index 2324cc32d09d..2bcb0ad79c35 100644 --- a/packages/jest-jasmine2/src/reporter.ts +++ b/packages/jest-jasmine2/src/reporter.ts @@ -141,6 +141,7 @@ export default class Jasmine2Reporter implements Reporter { const results: AssertionResult = { ancestorTitles, duration, + failureDetails: [], failureMessages: [], fullName: specResult.fullName, location, @@ -155,6 +156,7 @@ export default class Jasmine2Reporter implements Reporter { ? this._addMissingMessageToStack(failed.stack, failed.message) : failed.message || ''; results.failureMessages.push(message); + results.failureDetails.push(failed); }); return results; diff --git a/packages/jest-types/src/Circus.ts b/packages/jest-types/src/Circus.ts index c1ef85c79892..9868b5268122 100644 --- a/packages/jest-types/src/Circus.ts +++ b/packages/jest-types/src/Circus.ts @@ -158,10 +158,18 @@ export type AsyncEvent = name: 'teardown'; }; +export type MatcherResults = { + actual: unknown; + expected: unknown; + name: string; + pass: boolean; +}; + export type TestStatus = 'skip' | 'done' | 'todo'; export type TestResult = { duration?: number | null; errors: Array; + errorsDetailed: Array; invocations: number; status: TestStatus; location?: {column: number; line: number} | null; diff --git a/packages/jest-types/src/TestResult.ts b/packages/jest-types/src/TestResult.ts index 7b2c16b47b86..c8b5f123a1c9 100644 --- a/packages/jest-types/src/TestResult.ts +++ b/packages/jest-types/src/TestResult.ts @@ -18,6 +18,7 @@ type Callsite = { export type AssertionResult = { ancestorTitles: Array; duration?: Milliseconds | null; + failureDetails: Array; failureMessages: Array; fullName: string; invocations?: number;