diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 34fab62a57e9..128de1ee1777 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -1,38 +1,62 @@ import { existsSync, promises as fs } from 'fs' import { dirname, resolve } from 'pathe' import type { Vitest } from '../../node' -import type { File, Reporter } from '../../types' +import type { File, Reporter, Suite, TaskState } from '../../types' import { getSuites, getTests } from '../../utils' // for compatibility reasons, the reporter produces a JSON similar to the one produced by the Jest JSON reporter // the following types are extracted from the Jest repository (and simplified) +// the commented-out fields are the missing ones + +type Status = 'passed' | 'failed' | 'skipped' | 'pending' | 'todo' | 'disabled' type Milliseconds = number -interface TestResult { - displayName?: string - failureMessage?: string | null - skipped: boolean - status?: string - testFilePath?: string - perfStats: { - end?: Milliseconds - runtime?: Milliseconds - start?: Milliseconds - } +const StatusMap: Record = { + fail: 'failed', + only: 'pending', + pass: 'passed', + run: 'pending', + skip: 'skipped', + todo: 'todo', } -interface AggregatedResult { +interface FormattedAssertionResult { + ancestorTitles: Array + fullName: string + status: Status + title: string + duration?: Milliseconds | null + failureMessages: Array + // location?: Callsite | null +} + +interface FormattedTestResult { + message: string + name: string + status: 'failed' | 'passed' + startTime: number + endTime: number + assertionResults: Array + // summary: string + // coverage: unknown +} + +interface FormattedTestResults { numFailedTests: number numFailedTestSuites: number numPassedTests: number numPassedTestSuites: number numPendingTests: number - numTodoTests: number numPendingTestSuites: number + numTodoTests: number numTotalTests: number numTotalTestSuites: number startTime: number success: boolean - testResults: Array + testResults: Array + // coverageMap?: CoverageMap | null | undefined + // numRuntimeErrorTestSuites: number + // snapshot: SnapshotSummary + // wasInterrupted: boolean } export class JsonReporter implements Reporter { @@ -49,7 +73,6 @@ export class JsonReporter implements Reporter { const numTotalTestSuites = suites.length const tests = getTests(files) const numTotalTests = tests.length - const numFailedTestSuites = suites.filter(s => s.result?.error).length const numPassedTestSuites = numTotalTestSuites - numFailedTestSuites const numPendingTestSuites = suites.filter(s => s.result?.state === 'run').length @@ -57,23 +80,71 @@ export class JsonReporter implements Reporter { const numPassedTests = numTotalTests - numFailedTests const numPendingTests = tests.filter(t => t.result?.state === 'run').length const numTodoTests = tests.filter(t => t.mode === 'todo').length + const testResults: Array = [] const success = numFailedTestSuites === 0 && numFailedTests === 0 - const testResults: Array = tests.map(t => ({ - perfStats: { - runtime: t.result?.duration, - start: t.result?.startTime, - end: t.result?.duration && t.result?.startTime && t.result.duration + t.result.startTime, - }, - displayName: t.name, - failureMessage: t.result?.error?.message, - skipped: t.mode === 'skip', - status: t.result?.state, - testFilePath: t.file?.filepath, - })) - - const result: AggregatedResult = { numTotalTestSuites, numPassedTestSuites, numFailedTestSuites, numPendingTestSuites, numTotalTests, numPassedTests, numFailedTests, numPendingTests, numTodoTests, startTime: this.start, success, testResults } + for (const file of files) { + const tests = getTests([file]) + let startTime = tests.reduce((prev, next) => Math.min(prev, next.result?.startTime ?? Infinity), Infinity) + if (startTime === Infinity) + startTime = this.start + + const endTime = tests.reduce((prev, next) => Math.max(prev, (next.result?.startTime ?? 0) + (next.result?.duration ?? 0)), startTime) + const assertionResults = tests.map((t) => { + const ancestorTitles = [] as string[] + let iter: Suite | undefined = t.suite + while (iter) { + ancestorTitles.push(iter.name) + iter = iter.suite + } + ancestorTitles.reverse() + + return { + ancestorTitles, + fullName: ancestorTitles.length > 0 ? `${ancestorTitles.join(' ')} ${t.name}` : t.name, + status: t.result != null ? StatusMap[t.result.state] : 'skipped', + title: t.name, + duration: t.result?.duration, + failureMessages: t.result?.error?.message == null ? [] : [t.result.error.message], + } as FormattedAssertionResult + }) + + if (tests.some(t => t.result?.state === 'run')) { + this.ctx.console.warn('WARNING: Some tests are still running when generating the JSON report.' + + 'This is likely an internal bug in Vitest.' + + 'Please report it to https://github.com/vitest-dev/vitest/issues') + } + + testResults.push({ + assertionResults, + startTime, + endTime, + status: tests.every(t => + t.result?.state === 'pass' + || t.result?.state === 'skip' + || t.result?.state === 'todo') + ? 'passed' + : 'failed', + message: file.result?.error?.message ?? '', + name: file.filepath, + }) + } + + const result: FormattedTestResults = { + numTotalTestSuites, + numPassedTestSuites, + numFailedTestSuites, + numPendingTestSuites, + numTotalTests, + numPassedTests, + numFailedTests, + numPendingTests, + numTodoTests, + startTime: this.start, + success, + testResults, + } await this.writeReport(JSON.stringify(result, null, 2)) } diff --git a/test/reporters/tests/__snapshots__/reporters.spec.ts.snap b/test/reporters/tests/__snapshots__/reporters.spec.ts.snap index 4d6aa52003c5..5849b52817f2 100644 --- a/test/reporters/tests/__snapshots__/reporters.spec.ts.snap +++ b/test/reporters/tests/__snapshots__/reporters.spec.ts.snap @@ -115,74 +115,95 @@ exports[`json reporter 1`] = ` "success": false, "testResults": [ { - "displayName": "Math.sqrt()", - "failureMessage": "expected 2.23606797749979 to equal 2", - "perfStats": { - "runtime": 1.4422860145568848, - }, - "skipped": false, - "status": "fail", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "JSON", - "perfStats": { - "runtime": 1.0237109661102295, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "async with timeout", - "perfStats": {}, - "skipped": true, - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "timeout", - "perfStats": { - "runtime": 100.50598406791687, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "callback setup success ", - "perfStats": { - "runtime": 20.184875011444092, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "callback test success ", - "perfStats": { - "runtime": 0.33245420455932617, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "callback setup success done(false)", - "perfStats": { - "runtime": 19.738605976104736, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", - }, - { - "displayName": "callback test success done(false)", - "perfStats": { - "runtime": 0.1923508644104004, - }, - "skipped": false, - "status": "pass", - "testFilePath": "/vitest/test/core/test/basic.test.ts", + "assertionResults": [ + { + "ancestorTitles": [ + "suite", + "inner suite", + ], + "duration": 1.4422860145568848, + "failureMessages": [ + "expected 2.23606797749979 to equal 2", + ], + "fullName": "suite inner suite Math.sqrt()", + "status": "failed", + "title": "Math.sqrt()", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 1.0237109661102295, + "failureMessages": [], + "fullName": "suite JSON", + "status": "passed", + "title": "JSON", + }, + { + "ancestorTitles": [ + "suite", + ], + "failureMessages": [], + "fullName": "suite async with timeout", + "status": "skipped", + "title": "async with timeout", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 100.50598406791687, + "failureMessages": [], + "fullName": "suite timeout", + "status": "passed", + "title": "timeout", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 20.184875011444092, + "failureMessages": [], + "fullName": "suite callback setup success ", + "status": "passed", + "title": "callback setup success ", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 0.33245420455932617, + "failureMessages": [], + "fullName": "suite callback test success ", + "status": "passed", + "title": "callback test success ", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 19.738605976104736, + "failureMessages": [], + "fullName": "suite callback setup success done(false)", + "status": "passed", + "title": "callback setup success done(false)", + }, + { + "ancestorTitles": [ + "suite", + ], + "duration": 0.1923508644104004, + "failureMessages": [], + "fullName": "suite callback test success done(false)", + "status": "passed", + "title": "callback test success done(false)", + }, + ], + "endTime": 1642587001759, + "message": "", + "name": "/vitest/test/core/test/basic.test.ts", + "startTime": 1642587001759, + "status": "failed", }, ], } @@ -208,74 +229,95 @@ exports[`json reporter with outputFile 2`] = ` \\"success\\": false, \\"testResults\\": [ { - \\"perfStats\\": { - \\"runtime\\": 1.4422860145568848 - }, - \\"displayName\\": \\"Math.sqrt()\\", - \\"failureMessage\\": \\"expected 2.23606797749979 to equal 2\\", - \\"skipped\\": false, - \\"status\\": \\"fail\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 1.0237109661102295 - }, - \\"displayName\\": \\"JSON\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": {}, - \\"displayName\\": \\"async with timeout\\", - \\"skipped\\": true, - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 100.50598406791687 - }, - \\"displayName\\": \\"timeout\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 20.184875011444092 - }, - \\"displayName\\": \\"callback setup success \\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 0.33245420455932617 - }, - \\"displayName\\": \\"callback test success \\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 19.738605976104736 - }, - \\"displayName\\": \\"callback setup success done(false)\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 0.1923508644104004 - }, - \\"displayName\\": \\"callback test success done(false)\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" + \\"assertionResults\\": [ + { + \\"ancestorTitles\\": [ + \\"suite\\", + \\"inner suite\\" + ], + \\"fullName\\": \\"suite inner suite Math.sqrt()\\", + \\"status\\": \\"failed\\", + \\"title\\": \\"Math.sqrt()\\", + \\"duration\\": 1.4422860145568848, + \\"failureMessages\\": [ + \\"expected 2.23606797749979 to equal 2\\" + ] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite JSON\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"JSON\\", + \\"duration\\": 1.0237109661102295, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite async with timeout\\", + \\"status\\": \\"skipped\\", + \\"title\\": \\"async with timeout\\", + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite timeout\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"timeout\\", + \\"duration\\": 100.50598406791687, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback setup success \\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback setup success \\", + \\"duration\\": 20.184875011444092, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback test success \\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback test success \\", + \\"duration\\": 0.33245420455932617, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback setup success done(false)\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback setup success done(false)\\", + \\"duration\\": 19.738605976104736, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback test success done(false)\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback test success done(false)\\", + \\"duration\\": 0.1923508644104004, + \\"failureMessages\\": [] + } + ], + \\"startTime\\": 1642587001759, + \\"endTime\\": 1642587001759, + \\"status\\": \\"failed\\", + \\"message\\": \\"\\", + \\"name\\": \\"/vitest/test/core/test/basic.test.ts\\" } ] }" @@ -301,74 +343,95 @@ exports[`json reporter with outputFile in non-existing directory 2`] = ` \\"success\\": false, \\"testResults\\": [ { - \\"perfStats\\": { - \\"runtime\\": 1.4422860145568848 - }, - \\"displayName\\": \\"Math.sqrt()\\", - \\"failureMessage\\": \\"expected 2.23606797749979 to equal 2\\", - \\"skipped\\": false, - \\"status\\": \\"fail\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 1.0237109661102295 - }, - \\"displayName\\": \\"JSON\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": {}, - \\"displayName\\": \\"async with timeout\\", - \\"skipped\\": true, - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 100.50598406791687 - }, - \\"displayName\\": \\"timeout\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 20.184875011444092 - }, - \\"displayName\\": \\"callback setup success \\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 0.33245420455932617 - }, - \\"displayName\\": \\"callback test success \\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 19.738605976104736 - }, - \\"displayName\\": \\"callback setup success done(false)\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" - }, - { - \\"perfStats\\": { - \\"runtime\\": 0.1923508644104004 - }, - \\"displayName\\": \\"callback test success done(false)\\", - \\"skipped\\": false, - \\"status\\": \\"pass\\", - \\"testFilePath\\": \\"/vitest/test/core/test/basic.test.ts\\" + \\"assertionResults\\": [ + { + \\"ancestorTitles\\": [ + \\"suite\\", + \\"inner suite\\" + ], + \\"fullName\\": \\"suite inner suite Math.sqrt()\\", + \\"status\\": \\"failed\\", + \\"title\\": \\"Math.sqrt()\\", + \\"duration\\": 1.4422860145568848, + \\"failureMessages\\": [ + \\"expected 2.23606797749979 to equal 2\\" + ] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite JSON\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"JSON\\", + \\"duration\\": 1.0237109661102295, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite async with timeout\\", + \\"status\\": \\"skipped\\", + \\"title\\": \\"async with timeout\\", + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite timeout\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"timeout\\", + \\"duration\\": 100.50598406791687, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback setup success \\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback setup success \\", + \\"duration\\": 20.184875011444092, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback test success \\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback test success \\", + \\"duration\\": 0.33245420455932617, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback setup success done(false)\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback setup success done(false)\\", + \\"duration\\": 19.738605976104736, + \\"failureMessages\\": [] + }, + { + \\"ancestorTitles\\": [ + \\"suite\\" + ], + \\"fullName\\": \\"suite callback test success done(false)\\", + \\"status\\": \\"passed\\", + \\"title\\": \\"callback test success done(false)\\", + \\"duration\\": 0.1923508644104004, + \\"failureMessages\\": [] + } + ], + \\"startTime\\": 1642587001759, + \\"endTime\\": 1642587001759, + \\"status\\": \\"failed\\", + \\"message\\": \\"\\", + \\"name\\": \\"/vitest/test/core/test/basic.test.ts\\" } ] }"