feat(coverage): add reportOnFailure option (#3453)

Co-authored-by: Vladimir <sleuths.slews0s@icloud.com>

Author: AriPerkkio

docs/config/index.md

@@ -778,6 +778,16 @@ The reporter has three different types:
 
 Since Vitest 0.31.0, you can check your coverage report in Vitest UI: check [Vitest UI Coverage](/guide/coverage#vitest-ui) for more details.
 
+#### coverage.reportOnFailure
+
+- **Type:** `boolean`
+- **Default:** `true`
+- **Available for providers:** `'c8' | 'istanbul'`
+- **CLI:** `--coverage.reportOnFailure`, `--coverage.reportOnFailure=false`
+- **Version:** Since Vitest 0.31.2
+
+Generate coverage report even when tests fail.
+
 #### coverage.skipFull
 
 - **Type:** `boolean`

packages/vitest/src/defaults.ts

@@ -33,6 +33,7 @@ export const coverageConfigDefaults: ResolvedCoverageOptions = {
   cleanOnRerun: true,
   reportsDirectory: './coverage',
   exclude: defaultCoverageExcludes,
+  reportOnFailure: true,
   reporter: [['text', {}], ['html', {}], ['clover', {}], ['json', {}]],
   // default extensions used by c8, plus '.vue' and '.svelte'
   // see https://github.com/istanbuljs/schema/blob/master/default-extension.js

packages/vitest/src/node/core.ts

@@ -692,6 +692,9 @@ export class Vitest {
   }
 
   private async reportCoverage(allTestsRun: boolean) {
+    if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
+      return
+
     if (this.coverageProvider) {
       this.logger.log(c.blue(' % ') + c.dim('Coverage report from ') + c.yellow(this.coverageProvider.name))
       await this.coverageProvider.reportCoverage({ allTestsRun })

packages/vitest/src/types/coverage.ts

@@ -77,6 +77,7 @@ type FieldsWithDefaultValues =
   | 'reportsDirectory'
   | 'exclude'
   | 'extension'
+  | 'reportOnFailure'
 
 export type ResolvedCoverageOptions<T extends Provider = Provider> =
   & CoverageOptions<T>
@@ -208,6 +209,13 @@ export interface BaseCoverageOptions {
    * @default false
    */
   thresholdAutoUpdate?: boolean
+
+  /**
+   * Generate coverage report even when tests fail.
+   *
+   * @default true
+   */
+  reportOnFailure?: boolean
 }
 
 export interface CoverageIstanbulOptions extends BaseCoverageOptions {

pnpm-lock.yaml

@@ -105,6 +105,7 @@ test('provider module', () => {
             extension: ['string'],
             reporter: [['html', {}], ['json', { file: 'string' }]],
             reportsDirectory: 'string',
+            reportOnFailure: true,
           }
         },
         clean(_: boolean) {},

test/coverage-test/test/configuration-options.test-d.ts

@@ -6,6 +6,7 @@
     "coverage": "vitest run --coverage"
   },
   "devDependencies": {
+    "@vitest/coverage-istanbul": "workspace:*",
     "jsdom": "^21.0.0",
     "vitest": "workspace:*"
   }

test/fails/package.json

@@ -1,44 +1,44 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
-exports[`should fails > .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
+exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
 
-exports[`should fails > each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
+exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
 
-exports[`should fails > empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;
+exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;
 
-exports[`should fails > expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`;
+exports[`should fail expect.test.ts > expect.test.ts 1`] = `"AssertionError: expected 2 to deeply equal 3"`;
 
-exports[`should fails > hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`;
+exports[`should fail hook-timeout.test.ts > hook-timeout.test.ts 1`] = `"Error: Hook timed out in 10ms."`;
 
-exports[`should fails > hooks-called.test.ts > hooks-called.test.ts 1`] = `
+exports[`should fail hooks-called.test.ts > hooks-called.test.ts 1`] = `
 "Error: after all
 Error: before all"
 `;
 
-exports[`should fails > inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = `
+exports[`should fail inline-snapshop-inside-each.test.ts > inline-snapshop-inside-each.test.ts 1`] = `
 "Error: InlineSnapshot cannot be used inside of test.each or describe.each
 Error: InlineSnapshot cannot be used inside of test.each or describe.each
 Error: InlineSnapshot cannot be used inside of test.each or describe.each
 Error: InlineSnapshot cannot be used inside of test.each or describe.each
 Error: InlineSnapshot cannot be used inside of test.each or describe.each"
 `;
 
-exports[`should fails > mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`;
+exports[`should fail mock-import-proxy-module.test.ts > mock-import-proxy-module.test.ts 1`] = `"Error: There are some problems in resolving the mocks API."`;
 
-exports[`should fails > nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
+exports[`should fail nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
 
-exports[`should fails > primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`;
+exports[`should fail primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`;
 
-exports[`should fails > stall.test.ts > stall.test.ts 1`] = `
+exports[`should fail stall.test.ts > stall.test.ts 1`] = `
 "TypeError: failure
 TypeError: failure
 TypeError: failure
 TypeError: failure"
 `;
 
-exports[`should fails > test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
+exports[`should fail test-timeout.test.ts > test-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
 
-exports[`should fails > unhandled.test.ts > unhandled.test.ts 1`] = `
+exports[`should fail unhandled.test.ts > unhandled.test.ts 1`] = `
 "Error: some error
 Error: Uncaught [Error: some error]"
 `;

test/fails/test/__snapshots__/runner.test.ts.snap

@@ -1,25 +1,49 @@
 import { resolve } from 'pathe'
 import fg from 'fast-glob'
-import { describe, expect, it } from 'vitest'
+import { expect, it } from 'vitest'
 
 import { runVitest } from '../../test-utils'
 
-describe('should fails', async () => {
-  const root = resolve(__dirname, '../fixtures')
-  const files = await fg('**/*.test.ts', { cwd: root, dot: true })
-
-  for (const file of files) {
-    it(file, async () => {
-      const { stderr } = await runVitest({ root }, [file])
-
-      expect(stderr).toBeTruthy()
-      const msg = String(stderr)
-        .split(/\n/g)
-        .reverse()
-        .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest'))
-        .map(i => i.trim().replace(root, '<rootDir>'),
-        ).join('\n')
-      expect(msg).toMatchSnapshot(file)
-    }, 30_000)
-  }
+const root = resolve(__dirname, '../fixtures')
+const files = await fg('**/*.test.ts', { cwd: root, dot: true })
+
+it.each(files)('should fail %s', async (file) => {
+  const { stderr } = await runVitest({ root }, [file])
+
+  expect(stderr).toBeTruthy()
+  const msg = String(stderr)
+    .split(/\n/g)
+    .reverse()
+    .filter(i => i.includes('Error: ') && !i.includes('Command failed') && !i.includes('stackStr') && !i.includes('at runTest'))
+    .map(i => i.trim().replace(root, '<rootDir>'),
+    ).join('\n')
+  expect(msg).toMatchSnapshot(file)
+}, 30_000)
+
+it('should report coverage when "coverag.reportOnFailure: true" and tests fail', async () => {
+  const { stdout } = await runVitest({
+    root,
+    coverage: {
+      enabled: true,
+      provider: 'istanbul',
+      reportOnFailure: true,
+      reporter: ['text'],
+    },
+  }, [files[0]])
+
+  expect(stdout).toMatch('Coverage report from istanbul')
+})
+
+it('should not report coverage when "coverag.reportOnFailure: false" and tests fail', async () => {
+  const { stdout } = await runVitest({
+    root,
+    coverage: {
+      enabled: true,
+      provider: 'istanbul',
+      reportOnFailure: false,
+      reporter: ['text'],
+    },
+  }, [files[0]])
+
+  expect(stdout).not.toMatch('Coverage report from istanbul')
 })