fix(coverage): custom providers to work inside worker threads (#2817)

Author: AriPerkkio

docs/config/index.md

@@ -617,7 +617,7 @@ Isolate environment for each test file. Does not work if you disable [`--threads
 
 ### coverage
 
-You can use [`c8`](https://github.com/bcoe/c8) or [`istanbul`](https://istanbul.js.org/) for coverage collection.
+You can use [`c8`](https://github.com/bcoe/c8), [`istanbul`](https://istanbul.js.org/)  or [a custom coverage solution](/guide/coverage#custom-coverage-provider) for coverage collection.
 
 You can provide coverage options to CLI with dot notation:
 
@@ -631,7 +631,7 @@ If you are using coverage options with dot notation, don't forget to specify `--
 
 #### provider
 
-- **Type:** `'c8' | 'istanbul'`
+- **Type:** `'c8' | 'istanbul' | 'custom'`
 - **Default:** `'c8'`
 - **CLI:** `--coverage.provider=<provider>`
 
@@ -863,6 +863,14 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#ignoring-methods)
 
 Watermarks for statements, lines, branches and functions. See [istanbul documentation](https://github.com/istanbuljs/nyc#high-and-low-watermarks) for more information.
 
+#### customProviderModule
+
+- **Type:** `string`
+- **Available for providers:** `'custom'`
+- **CLI:** `--coverage.customProviderModule=<path or module name>`
+
+Specifies the module name or path for the custom coverage provider module. See [Guide - Custom Coverage Provider](/guide/coverage#custom-coverage-provider) for more information.
+
 ### testNamePattern
 
 - **Type** `string | RegExp`

docs/guide/coverage.md

@@ -71,22 +71,50 @@ export default defineConfig({
 
 ## Custom Coverage Provider
 
-It's also possible to provide your custom coverage provider by passing an object to the `test.coverage.provider`:
+It's also possible to provide your custom coverage provider by passing `'custom'` in `test.coverage.provider`:
 
 ```ts
 // vite.config.ts
 import { defineConfig } from 'vitest/config'
-import CustomCoverageProvider from 'my-custom-coverage-provider'
 
 export default defineConfig({
   test: {
     coverage: {
-      provider: CustomCoverageProvider()
+      provider: 'custom',
+      customProviderModule: 'my-custom-coverage-provider'
     },
   },
 })
 ```
 
+The custom providers require a `customProviderModule` option which is a module name or path where to load the `CoverageProviderModule` from. It must export an object that implements `CoverageProviderModule` as default export:
+
+```ts
+// my-custom-coverage-provider.ts
+import type { CoverageProvider, CoverageProviderModule, ResolvedCoverageOptions, Vitest } from 'vitest'
+
+const CustomCoverageProviderModule: CoverageProviderModule = {
+  getProvider(): CoverageProvider {
+    return new CustomCoverageProvider()
+  },
+
+  // Implements rest of the CoverageProviderModule ...
+}
+
+class CustomCoverageProvider implements CoverageProvider {
+  name = 'custom-coverage-provider'
+  options!: ResolvedCoverageOptions
+
+  initialize(ctx: Vitest) {
+    this.options = ctx.config.coverage
+  }
+
+  // Implements rest of the CoverageProvider ...
+}
+
+export default CustomCoverageProviderModule
+```
+
 Please refer to the type definition for more details.
 
 ## Changing the default coverage folder location

packages/vitest/src/integrations/coverage.ts

@@ -1,34 +1,47 @@
 import { importModule } from 'local-pkg'
 import type { CoverageOptions, CoverageProvider, CoverageProviderModule } from '../types'
 
-export const CoverageProviderMap = {
+interface Loader {
+  executeId: (id: string) => Promise<{ default: CoverageProviderModule }>
+}
+
+export const CoverageProviderMap: Record<string, string> = {
   c8: '@vitest/coverage-c8',
   istanbul: '@vitest/coverage-istanbul',
 }
 
-export async function resolveCoverageProvider(provider: NonNullable<CoverageOptions['provider']>) {
-  if (typeof provider === 'string') {
-    const pkg = CoverageProviderMap[provider]
-    if (!pkg)
-      throw new Error(`Unknown coverage provider: ${provider}`)
-    return await importModule<CoverageProviderModule>(pkg)
+async function resolveCoverageProviderModule(options: CoverageOptions & Required<Pick<CoverageOptions, 'provider'>>, loader: Loader) {
+  const provider = options.provider
+
+  if (provider === 'c8' || provider === 'istanbul')
+    return await importModule<CoverageProviderModule>(CoverageProviderMap[provider])
+
+  let customProviderModule
+
+  try {
+    customProviderModule = await loader.executeId(options.customProviderModule)
   }
-  else {
-    return provider
+  catch (error) {
+    throw new Error(`Failed to load custom CoverageProviderModule from ${options.customProviderModule}`, { cause: error })
   }
+
+  if (customProviderModule.default == null)
+    throw new Error(`Custom CoverageProviderModule loaded from ${options.customProviderModule} was not the default export`)
+
+  return customProviderModule.default
 }
 
-export async function getCoverageProvider(options?: CoverageOptions): Promise<CoverageProvider | null> {
-  if (options?.enabled && options?.provider) {
-    const { getProvider } = await resolveCoverageProvider(options.provider)
+export async function getCoverageProvider(options: CoverageOptions, loader: Loader): Promise<CoverageProvider | null> {
+  if (options.enabled && options.provider) {
+    const { getProvider } = await resolveCoverageProviderModule(options, loader)
     return await getProvider()
   }
   return null
 }
 
-export async function takeCoverageInsideWorker(options: CoverageOptions) {
+export async function takeCoverageInsideWorker(options: CoverageOptions, loader: Loader) {
   if (options.enabled && options.provider) {
-    const { takeCoverage } = await resolveCoverageProvider(options.provider)
+    const { takeCoverage } = await resolveCoverageProviderModule(options, loader)
     return await takeCoverage?.()
   }
 }

packages/vitest/src/node/cli-api.ts

@@ -50,9 +50,9 @@ export async function startVitest(
 
   if (mode === 'test' && ctx.config.coverage.enabled) {
     const provider = ctx.config.coverage.provider || 'c8'
-    if (typeof provider === 'string') {
-      const requiredPackages = CoverageProviderMap[provider]
+    const requiredPackages = CoverageProviderMap[provider]
 
+    if (requiredPackages) {
       if (!await ensurePackageInstalled(requiredPackages, root)) {
         process.exitCode = 1
         return ctx

packages/vitest/src/node/core.ts

@@ -124,7 +124,7 @@ export class Vitest {
   async initCoverageProvider() {
     if (this.coverageProvider !== undefined)
       return
-    this.coverageProvider = await getCoverageProvider(this.config.coverage)
+    this.coverageProvider = await getCoverageProvider(this.config.coverage, this.runner)
     if (this.coverageProvider) {
       await this.coverageProvider.initialize(this)
       this.config.coverage = this.coverageProvider.resolveOptions()

packages/vitest/src/runtime/entry.ts

@@ -68,7 +68,7 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor):
 
   const originalOnAfterRun = testRunner.onAfterRun
   testRunner.onAfterRun = async (files) => {
-    const coverage = await takeCoverageInsideWorker(config.coverage)
+    const coverage = await takeCoverageInsideWorker(config.coverage, executor)
     rpc().onAfterSuiteRun({ coverage })
     await originalOnAfterRun?.call(testRunner, files)
   }

packages/vitest/src/types/coverage.ts

@@ -53,12 +53,13 @@ export type CoverageReporter =
   | 'text-summary'
   | 'text'
 
-type Provider = 'c8' | 'istanbul' | CoverageProviderModule | undefined
+type Provider = 'c8' | 'istanbul' | 'custom' | undefined
 
 export type CoverageOptions<T extends Provider = Provider> =
-  T extends CoverageProviderModule ? ({ provider: T } & BaseCoverageOptions) :
-    T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
-        ({ provider?: T } & CoverageC8Options)
+  T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) :
+    T extends 'c8' ? ({ provider: T } & CoverageC8Options) :
+      T extends 'custom' ? ({ provider: T } & CustomProviderOptions) :
+          ({ provider?: T } & (CoverageC8Options))
 
 /** Fields that have default values. Internally these will always be defined. */
 type FieldsWithDefaultValues =
@@ -233,3 +234,8 @@ export interface CoverageC8Options extends BaseCoverageOptions {
    */
   100?: boolean
 }
+
+export interface CustomProviderOptions extends Pick<BaseCoverageOptions, FieldsWithDefaultValues> {
+  /** Name of the module or path to a file to load the custom provider from */
+  customProviderModule: string
+}

test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap

@@ -0,0 +1,26 @@
+// Vitest Snapshot v1
+
+exports[`custom json report 1`] = `
+{
+  "calls": [
+    "initialized with context",
+    "resolveOptions",
+    "clean with force",
+    "onBeforeFilesRun",
+    "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
+    "reportCoverage with {\\"allTestsRun\\":true}",
+  ],
+  "transformedFiles": [
+    "<process-cwd>/src/Counter/Counter.component.ts",
+    "<process-cwd>/src/Counter/Counter.vue",
+    "<process-cwd>/src/Counter/index.ts",
+    "<process-cwd>/src/Defined.vue",
+    "<process-cwd>/src/Hello.vue",
+    "<process-cwd>/src/another-setup.ts",
+    "<process-cwd>/src/implicitElse.ts",
+    "<process-cwd>/src/importEnv.ts",
+    "<process-cwd>/src/index.mts",
+    "<process-cwd>/src/utils.ts",
+  ],
+}
+`;

test/coverage-test/coverage-report-tests/custom.report.test.ts

@@ -0,0 +1,12 @@
+/*
+ * Custom coverage provider specific test cases
+ */
+
+import { readFileSync } from 'fs'
+import { expect, test } from 'vitest'
+
+test('custom json report', async () => {
+  const report = readFileSync('./coverage/custom-coverage-provider-report.json', 'utf-8')
+
+  expect(JSON.parse(report)).toMatchSnapshot()
+})

test/coverage-test/coverage-report-tests/utils.ts

@@ -1,3 +1,4 @@
+import { readFileSync } from 'fs'
 import { normalize } from 'pathe'
 
 interface CoverageFinalJson {
@@ -17,8 +18,7 @@ interface CoverageFinalJson {
  * Normalizes paths to keep contents consistent between OS's
  */
 export async function readCoverageJson() {
-  // @ts-expect-error -- generated file
-  const { default: jsonReport } = await import('./coverage/coverage-final.json') as CoverageFinalJson
+  const jsonReport = JSON.parse(readFileSync('./coverage/coverage-final.json', 'utf8')) as CoverageFinalJson
 
   const normalizedReport: CoverageFinalJson['default'] = {}
 
@@ -30,6 +30,6 @@ export async function readCoverageJson() {
   return normalizedReport
 }
 
-function normalizeFilename(filename: string) {
+export function normalizeFilename(filename: string) {
   return normalize(filename).replace(normalize(process.cwd()), '<process-cwd>')
 }

test/coverage-test/custom-provider.ts

@@ -0,0 +1,75 @@
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs'
+import type { AfterSuiteRunMeta, CoverageProvider, CoverageProviderModule, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
+
+import { normalizeFilename } from './coverage-report-tests/utils'
+
+const CustomCoverageProviderModule: CoverageProviderModule = {
+  getProvider(): CoverageProvider {
+    return new CustomCoverageProvider()
+  },
+
+  takeCoverage() {
+    return { customCoverage: 'Coverage report passed from workers to main thread' }
+  },
+}
+
+/**
+ * Provider that simply keeps track of the functions that were called
+ */
+class CustomCoverageProvider implements CoverageProvider {
+  name = 'custom-coverage-provider'
+
+  options!: ResolvedCoverageOptions
+  calls: Set<string> = new Set()
+  transformedFiles: Set<string> = new Set()
+
+  initialize(ctx: Vitest) {
+    this.options = ctx.config.coverage
+
+    this.calls.add(`initialized ${ctx ? 'with' : 'without'} context`)
+  }
+
+  clean(force: boolean) {
+    this.calls.add(`clean ${force ? 'with' : 'without'} force`)
+  }
+
+  onBeforeFilesRun() {
+    this.calls.add('onBeforeFilesRun')
+  }
+
+  onAfterSuiteRun(meta: AfterSuiteRunMeta) {
+    this.calls.add(`onAfterSuiteRun with ${JSON.stringify(meta)}`)
+  }
+
+  reportCoverage(reportContext?: ReportContext) {
+    this.calls.add(`reportCoverage with ${JSON.stringify(reportContext)}`)
+
+    const jsonReport = JSON.stringify({
+      calls: Array.from(this.calls.values()),
+      transformedFiles: Array.from(this.transformedFiles.values()).sort(),
+    }, null, 2)
+
+    if (existsSync('./coverage'))
+      rmSync('./coverage', { maxRetries: 10, recursive: true })
+
+    mkdirSync('./coverage')
+    writeFileSync('./coverage/custom-coverage-provider-report.json', jsonReport, 'utf-8')
+  }
+
+  onFileTransform(code: string, id: string) {
+    const filename = normalizeFilename(id).split('?')[0]
+
+    if (/\/src\//.test(filename))
+      this.transformedFiles.add(filename)
+
+    return { code }
+  }
+
+  resolveOptions(): ResolvedCoverageOptions {
+    this.calls.add('resolveOptions')
+
+    return this.options
+  }
+}
+
+export default CustomCoverageProviderModule

test/coverage-test/package.json

@@ -2,10 +2,11 @@
   "name": "@vitest/test-coverage",
   "private": true,
   "scripts": {
-    "test": "pnpm run test:c8 && pnpm run test:istanbul && pnpm run test:types",
+    "test": "pnpm test:c8 && pnpm test:istanbul && pnpm test:custom && pnpm test:types",
     "test:c8": "node ./testing.mjs --provider c8",
+    "test:custom": "node ./testing.mjs --provider custom",
     "test:istanbul": "node ./testing.mjs --provider istanbul",
-    "test:types": "vitest typecheck --run"
+    "test:types": "vitest typecheck --run --reporter verbose"
   },
   "devDependencies": {
     "@vitejs/plugin-vue": "latest",

test/coverage-test/test/configuration-options.test-d.ts

@@ -1,5 +1,5 @@
 import { assertType, test } from 'vitest'
-import type { ResolvedCoverageOptions, Vitest } from 'vitest'
+import type { CoverageProviderModule, ResolvedCoverageOptions, Vitest } from 'vitest'
 import type { defineConfig } from 'vitest/config'
 
 type NarrowToTestConfig<T> = T extends { test?: any } ? NonNullable<T['test']> : never
@@ -10,39 +10,14 @@ test('providers, built-in', () => {
   assertType<Coverage>({ provider: 'c8' })
   assertType<Coverage>({ provider: 'istanbul' })
 
-  // @ts-expect-error -- String options must be known built-in's
-  assertType<Coverage>({ provider: 'unknown-reporter' })
+  // @ts-expect-error -- String options must be known ones only
+  assertType<Coverage>({ provider: 'unknown-provider' })
 })
 
 test('providers, custom', () => {
   assertType<Coverage>({
-    provider: {
-      getProvider() {
-        return {
-          name: 'custom-provider',
-          initialize(_: Vitest) {},
-          resolveOptions(): ResolvedCoverageOptions {
-            return {
-              clean: true,
-              cleanOnRerun: true,
-              enabled: true,
-              exclude: ['string'],
-              extension: ['string'],
-              reporter: ['html', 'json'],
-              reportsDirectory: 'string',
-            }
-          },
-          clean(_: boolean) {},
-          onBeforeFilesRun() {},
-          onAfterSuiteRun({ coverage: _coverage }) {},
-          reportCoverage() {},
-          onFileTransform(_code: string, _id: string, ctx) {
-            ctx.getCombinedSourcemap()
-          },
-        }
-      },
-      takeCoverage() {},
-    },
+    provider: 'custom',
+    customProviderModule: 'custom-provider-module.ts',
   })
 })
 
@@ -95,6 +70,55 @@ test('provider specific options, istanbul', () => {
   })
 })
 
+test('provider specific options, custom', () => {
+  assertType<Coverage>({
+    provider: 'custom',
+    customProviderModule: 'custom-provider-module.ts',
+    enabled: true,
+  })
+
+  // @ts-expect-error --  customProviderModule is required
+  assertType<Coverage>({ provider: 'custom' })
+
+  assertType<Coverage>({
+    provider: 'custom',
+    customProviderModule: 'some-module',
+
+    // @ts-expect-error --  typings of BaseCoverageOptions still apply
+    enabled: 'not boolean',
+  })
+})
+
+test('provider module', () => {
+  assertType<CoverageProviderModule>({
+    getProvider() {
+      return {
+        name: 'custom-provider',
+        initialize(_: Vitest) {},
+        resolveOptions(): ResolvedCoverageOptions {
+          return {
+            clean: true,
+            cleanOnRerun: true,
+            enabled: true,
+            exclude: ['string'],
+            extension: ['string'],
+            reporter: ['html', 'json'],
+            reportsDirectory: 'string',
+          }
+        },
+        clean(_: boolean) {},
+        onBeforeFilesRun() {},
+        onAfterSuiteRun({ coverage: _coverage }) {},
+        reportCoverage() {},
+        onFileTransform(_code: string, _id: string, ctx) {
+          ctx.getCombinedSourcemap()
+        },
+      }
+    },
+    takeCoverage() {},
+  })
+})
+
 test('reporters, single', () => {
   assertType<Coverage>({ reporter: 'clover' })
   assertType<Coverage>({ reporter: 'cobertura' })

test/coverage-test/testing.mjs

@@ -3,61 +3,35 @@ import { startVitest } from 'vitest/node'
 // Set this to true when intentionally updating the snapshots
 const UPDATE_SNAPSHOTS = false
 
-const provider = getArgument('--provider')
+const provider = process.argv[1 + process.argv.indexOf('--provider')]
 
 const configs = [
   // Run test cases. Generates coverage report.
   ['test/', {
     include: ['test/*.test.*'],
     exclude: ['coverage-report-tests/**/*'],
+    coverage: { enabled: true },
   }],
 
   // Run tests for checking coverage report contents.
   ['coverage-report-tests', {
     include: [
-      './coverage-report-tests/generic.report.test.ts',
+      ['c8', 'istanbul'].includes(provider) && './coverage-report-tests/generic.report.test.ts',
       `./coverage-report-tests/${provider}.report.test.ts`,
-    ],
+    ].filter(Boolean),
     coverage: { enabled: false, clean: false },
   }],
 ]
 
-runTests()
+for (const threads of [true, false]) {
+  for (const [directory, config] of configs) {
+    await startVitest('test', [directory], {
+      ...config,
+      update: UPDATE_SNAPSHOTS,
+      threads,
+    })
 
-async function runTests() {
-  for (const threads of [true, false]) {
-    for (const [directory, config] of configs) {
-      await startVitest('test', [directory], {
-        run: true,
-        update: UPDATE_SNAPSHOTS,
-        ...config,
-        threads,
-        coverage: {
-          include: ['src/**'],
-          provider,
-          ...config.coverage,
-        },
-      })
-
-      if (process.exitCode)
-        process.exit()
-    }
+    if (process.exitCode)
+      process.exit()
   }
-
-  process.exit(0)
-}
-
-function getArgument(name) {
-  const args = process.argv
-  const index = args.indexOf(name)
-
-  if (index === -1)
-    throw new Error(`Missing argument ${name}, received ${args}`)
-
-  const value = args[index + 1]
-
-  if (!value)
-    throw new Error(`Missing value of ${name}, received ${args}`)
-
-  return value
 }

test/coverage-test/vitest.config.ts

@@ -2,6 +2,8 @@ import { resolve } from 'pathe'
 import { defineConfig } from 'vitest/config'
 import vue from '@vitejs/plugin-vue'
 
+const provider = process.argv[1 + process.argv.indexOf('--provider')]
+
 export default defineConfig({
   plugins: [
     vue(),
@@ -10,9 +12,11 @@ export default defineConfig({
     MY_CONSTANT: '"my constant"',
   },
   test: {
-    reporters: 'verbose',
+    watch: false,
     coverage: {
-      enabled: true,
+      provider: provider as any,
+      customProviderModule: provider === 'custom' ? 'custom-provider' : undefined,
+      include: ['src/**'],
       clean: true,
       all: true,
       reporter: ['html', 'text', 'lcov', 'json'],