diff --git a/docs/config/index.md b/docs/config/index.md index 37ba61e626e6..f45d642f93ce 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -778,6 +778,16 @@ The reporter has three different types: Since Vitest 0.31.0, you can check your coverage report in Vitest UI: check [Vitest UI Coverage](/guide/coverage#vitest-ui) for more details. +#### coverage.reportOnFailure + +- **Type:** `boolean` +- **Default:** `true` +- **Available for providers:** `'c8' | 'istanbul'` +- **CLI:** `--coverage.reportOnFailure`, `--coverage.reportOnFailure=false` +- **Version:** Since Vitest 0.31.2 + +Generate coverage report even when tests fail. + #### coverage.skipFull - **Type:** `boolean` diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index f06e14bf4c65..0190f27ffb89 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -33,6 +33,7 @@ export const coverageConfigDefaults: ResolvedCoverageOptions = { cleanOnRerun: true, reportsDirectory: './coverage', exclude: defaultCoverageExcludes, + reportOnFailure: true, reporter: [['text', {}], ['html', {}], ['clover', {}], ['json', {}]], // default extensions used by c8, plus '.vue' and '.svelte' // see https://github.com/istanbuljs/schema/blob/master/default-extension.js diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index bdbe27f6bf0d..3cd67b33687c 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -692,6 +692,9 @@ export class Vitest { } private async reportCoverage(allTestsRun: boolean) { + if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0) + return + if (this.coverageProvider) { this.logger.log(c.blue(' % ') + c.dim('Coverage report from ') + c.yellow(this.coverageProvider.name)) await this.coverageProvider.reportCoverage({ allTestsRun }) diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index fe2d62562d18..65cf0d9e800b 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -77,6 +77,7 @@ type FieldsWithDefaultValues = | 'reportsDirectory' | 'exclude' | 'extension' + | 'reportOnFailure' export type ResolvedCoverageOptions = & CoverageOptions @@ -208,6 +209,13 @@ export interface BaseCoverageOptions { * @default false */ thresholdAutoUpdate?: boolean + + /** + * Generate coverage report even when tests fail. + * + * @default true + */ + reportOnFailure?: boolean } export interface CoverageIstanbulOptions extends BaseCoverageOptions { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 62fc2c88ac9a..6a02e76647b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1594,6 +1594,9 @@ importers: test/fails: devDependencies: + '@vitest/coverage-istanbul': + specifier: workspace:* + version: link:../../packages/coverage-istanbul jsdom: specifier: ^21.0.0 version: 21.0.0 diff --git a/test/coverage-test/test/configuration-options.test-d.ts b/test/coverage-test/test/configuration-options.test-d.ts index 8b9b0de67c99..d9f17e107fa6 100644 --- a/test/coverage-test/test/configuration-options.test-d.ts +++ b/test/coverage-test/test/configuration-options.test-d.ts @@ -105,6 +105,7 @@ test('provider module', () => { extension: ['string'], reporter: [['html', {}], ['json', { file: 'string' }]], reportsDirectory: 'string', + reportOnFailure: true, } }, clean(_: boolean) {}, diff --git a/test/fails/package.json b/test/fails/package.json index fe036ef4e339..3635f317299d 100644 --- a/test/fails/package.json +++ b/test/fails/package.json @@ -6,6 +6,7 @@ "coverage": "vitest run --coverage" }, "devDependencies": { + "@vitest/coverage-istanbul": "workspace:*", "jsdom": "^21.0.0", "vitest": "workspace:*" } diff --git a/test/fails/test/__snapshots__/runner.test.ts.snap b/test/fails/test/__snapshots__/runner.test.ts.snap index 66dbe981fe39..867a4673307f 100644 --- a/test/fails/test/__snapshots__/runner.test.ts.snap +++ b/test/fails/test/__snapshots__/runner.test.ts.snap @@ -1,21 +1,21 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`should fails > .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; +exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; -exports[`should fails > each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; +exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; -exports[`should fails > empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file /empty.test.ts"`; +exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file /empty.test.ts"`; -exports[`should fails > expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`; +exports[`should fail expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`; -exports[`should fails > hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`; +exports[`should fail hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`; -exports[`should fails > hooks-called.test.ts > hooks-called.test.ts 1`] = ` +exports[`should fail hooks-called.test.ts > hooks-called.test.ts 1`] = ` "Error: after all Error: before all" `; -exports[`should fails > inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = ` +exports[`should fail inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = ` "Error: InlineSnapshot cannot be used inside of test.each or describe.each Error: InlineSnapshot cannot be used inside of test.each or describe.each Error: InlineSnapshot cannot be used inside of test.each or describe.each @@ -23,22 +23,22 @@ Error: InlineSnapshot cannot be used inside of test.each or describe.each Error: InlineSnapshot cannot be used inside of test.each or describe.each" `; -exports[`should fails > mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`; +exports[`should fail mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`; -exports[`should fails > nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; +exports[`should fail nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; -exports[`should fails > primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`; +exports[`should fail primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`; -exports[`should fails > stall.test.ts > stall.test.ts 1`] = ` +exports[`should fail stall.test.ts > stall.test.ts 1`] = ` "TypeError: failure TypeError: failure TypeError: failure TypeError: failure" `; -exports[`should fails > test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; +exports[`should fail test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`; -exports[`should fails > unhandled.test.ts > unhandled.test.ts 1`] = ` +exports[`should fail unhandled.test.ts > unhandled.test.ts 1`] = ` "Error: some error Error: Uncaught [Error: some error]" `; diff --git a/test/fails/test/runner.test.ts b/test/fails/test/runner.test.ts index 54ae49740be1..0132209d5e4c 100644 --- a/test/fails/test/runner.test.ts +++ b/test/fails/test/runner.test.ts @@ -1,25 +1,49 @@ import { resolve } from 'pathe' import fg from 'fast-glob' -import { describe, expect, it } from 'vitest' +import { expect, it } from 'vitest' import { runVitest } from '../../test-utils' -describe('should fails', async () => { - const root = resolve(__dirname, '../fixtures') - const files = await fg('**/*.test.ts', { cwd: root, dot: true }) - - for (const file of files) { - it(file, async () => { - const { stderr } = await runVitest({ root }, [file]) - - expect(stderr).toBeTruthy() - const msg = String(stderr) - .split(/\n/g) - .reverse() - .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest')) - .map(i => i.trim().replace(root, ''), - ).join('\n') - expect(msg).toMatchSnapshot(file) - }, 30_000) - } +const root = resolve(__dirname, '../fixtures') +const files = await fg('**/*.test.ts', { cwd: root, dot: true }) + +it.each(files)('should fail %s', async (file) => { + const { stderr } = await runVitest({ root }, [file]) + + expect(stderr).toBeTruthy() + const msg = String(stderr) + .split(/\n/g) + .reverse() + .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest')) + .map(i => i.trim().replace(root, ''), + ).join('\n') + expect(msg).toMatchSnapshot(file) +}, 30_000) + +it('should report coverage when "coverag.reportOnFailure: true" and tests fail', async () => { + const { stdout } = await runVitest({ + root, + coverage: { + enabled: true, + provider: 'istanbul', + reportOnFailure: true, + reporter: ['text'], + }, + }, [files[0]]) + + expect(stdout).toMatch('Coverage report from istanbul') +}) + +it('should not report coverage when "coverag.reportOnFailure: false" and tests fail', async () => { + const { stdout } = await runVitest({ + root, + coverage: { + enabled: true, + provider: 'istanbul', + reportOnFailure: false, + reporter: ['text'], + }, + }, [files[0]]) + + expect(stdout).not.toMatch('Coverage report from istanbul') })