feat!: coverage-c8 to use V8 profiler directly instead of NODE_V8_COVERAGE (#2786)

Author: AriPerkkio

docs/config/index.md

@@ -722,7 +722,6 @@ Clean coverage report on watch rerun
 - **CLI:** `--coverage.reportsDirectory=<path>`
 
 Directory to write coverage report to.
-When using `c8` provider a temporary `/tmp` directory is created for [V8 coverage results](https://nodejs.org/api/cli.html#coverage-output).
 
 #### reporter
 

packages/coverage-c8/package.json

@@ -42,7 +42,7 @@
     "prepublishOnly": "pnpm build"
   },
   "peerDependencies": {
-    "vitest": ">=0.28.0 <1"
+    "vitest": ">=0.29.0 <1"
   },
   "dependencies": {
     "c8": "^7.12.0",

packages/coverage-c8/src/provider.ts

@@ -1,30 +1,28 @@
 import { existsSync, promises as fs } from 'fs'
 import _url from 'url'
 import type { Profiler } from 'inspector'
-import { takeCoverage } from 'v8'
 import { extname, resolve } from 'pathe'
 import c from 'picocolors'
 import { provider } from 'std-env'
 import type { RawSourceMap } from 'vite-node'
 import { coverageConfigDefaults } from 'vitest/config'
 // eslint-disable-next-line no-restricted-imports
-import type { CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest'
+import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest'
 import type { Vitest } from 'vitest/node'
 import type { Report } from 'c8'
 // @ts-expect-error missing types
 import createReport from 'c8/lib/report.js'
 // @ts-expect-error missing types
 import { checkCoverages } from 'c8/lib/commands/check-coverage.js'
 
-type Options =
-  & ResolvedCoverageOptions<'c8'>
-  & { tempDirectory: string }
+type Options = ResolvedCoverageOptions<'c8'>
 
 export class C8CoverageProvider implements CoverageProvider {
   name = 'c8'
 
   ctx!: Vitest
   options!: Options
+  coverages: Profiler.TakePreciseCoverageReturnType[] = []
 
   initialize(ctx: Vitest) {
     this.ctx = ctx
@@ -35,25 +33,18 @@ export class C8CoverageProvider implements CoverageProvider {
     return this.options
   }
 
-  onBeforeFilesRun() {
-    process.env.NODE_V8_COVERAGE ||= this.options.tempDirectory
-  }
-
   async clean(clean = true) {
     if (clean && existsSync(this.options.reportsDirectory))
       await fs.rm(this.options.reportsDirectory, { recursive: true, force: true, maxRetries: 10 })
 
-    if (!existsSync(this.options.tempDirectory))
-      await fs.mkdir(this.options.tempDirectory, { recursive: true })
+    this.coverages = []
   }
 
-  onAfterSuiteRun() {
-    takeCoverage()
+  onAfterSuiteRun({ coverage }: AfterSuiteRunMeta) {
+    this.coverages.push(coverage as Profiler.TakePreciseCoverageReturnType)
   }
 
   async reportCoverage({ allTestsRun }: ReportContext = {}) {
-    takeCoverage()
-
     if (provider === 'stackblitz')
       this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-c8 does not work on Stackblitz. Report will be empty.'))
 
@@ -64,6 +55,9 @@ export class C8CoverageProvider implements CoverageProvider {
 
     const report = createReport(options)
 
+    // Overwrite C8's loader as results are in memory instead of file system
+    report._loadReports = () => this.coverages
+
     interface MapAndSource { map: RawSourceMap; source: string | undefined }
     type SourceMapMeta = { url: string; filepath: string } & MapAndSource
 
@@ -73,7 +67,7 @@ export class C8CoverageProvider implements CoverageProvider {
 
     const entries = Array
       .from(this.ctx.vitenode.fetchCache.entries())
-      .filter(i => !i[0].includes('/node_modules/'))
+      .filter(entry => report._shouldInstrument(entry[0]))
       .map(([file, { result }]) => {
         if (!result.map)
           return null
@@ -153,12 +147,6 @@ export class C8CoverageProvider implements CoverageProvider {
 
     await report.run()
     await checkCoverages(options, report)
-
-    // Note that this will only clean up the V8 reports generated so far.
-    // There will still be a temp directory with some reports when vitest exists,
-    // but at least it will only contain reports of vitest's internal functions.
-    if (existsSync(this.options.tempDirectory))
-      await fs.rm(this.options.tempDirectory, { recursive: true, force: true, maxRetries: 10 })
   }
 }
 
@@ -178,7 +166,6 @@ function resolveC8Options(options: CoverageC8Options, root: string): Options {
 
     // Resolved fields
     provider: 'c8',
-    tempDirectory: process.env.NODE_V8_COVERAGE || resolve(reportsDirectory, 'tmp'),
     reporter: Array.isArray(reporter) ? reporter : [reporter],
     reportsDirectory,
   }

packages/coverage-c8/src/takeCoverage.ts

@@ -1,10 +1,46 @@
-import v8 from 'v8'
+/*
+ * For details about the Profiler.* messages see https://chromedevtools.github.io/devtools-protocol/v8/Profiler/
+*/
 
-// Flush coverage to disk
+import inspector from 'node:inspector'
+import type { Profiler } from 'node:inspector'
 
-export function takeCoverage() {
-  if (v8.takeCoverage == null)
-    console.warn('[Vitest] takeCoverage is not available in this NodeJs version.\nCoverage could be incomplete. Update to NodeJs 14.18.')
-  else
-    v8.takeCoverage()
+const session = new inspector.Session()
+
+export function startCoverage() {
+  session.connect()
+  session.post('Profiler.enable')
+  session.post('Profiler.startPreciseCoverage', {
+    callCount: true,
+    detailed: true,
+  })
+}
+
+export async function takeCoverage() {
+  return new Promise((resolve, reject) => {
+    session.post('Profiler.takePreciseCoverage', async (error, coverage) => {
+      if (error)
+        return reject(error)
+
+      // Reduce amount of data sent over rpc by doing some early result filtering
+      const result = coverage.result.filter(filterResult)
+
+      resolve({ result })
+    })
+  })
+}
+
+export function stopCoverage() {
+  session.post('Profiler.stopPreciseCoverage')
+  session.post('Profiler.disable')
+}
+
+function filterResult(coverage: Profiler.ScriptCoverage): boolean {
+  if (!coverage.url.startsWith('file://'))
+    return false
+
+  if (coverage.url.includes('/node_modules/'))
+    return false
+
+  return true
 }

packages/vitest/src/integrations/coverage.ts

@@ -10,7 +10,10 @@ export const CoverageProviderMap: Record<string, string> = {
   istanbul: '@vitest/coverage-istanbul',
 }
 
-async function resolveCoverageProviderModule(options: CoverageOptions & Required<Pick<CoverageOptions, 'provider'>>, loader: Loader) {
+async function resolveCoverageProviderModule(options: CoverageOptions | undefined, loader: Loader) {
+  if (!options?.enabled || !options.provider)
+    return null
+
   const provider = options.provider
 
   if (provider === 'c8' || provider === 'istanbul')
@@ -31,17 +34,38 @@ async function resolveCoverageProviderModule(options: CoverageOptions & Required
   return customProviderModule.default
 }
 
-export async function getCoverageProvider(options: CoverageOptions, loader: Loader): Promise<CoverageProvider | null> {
-  if (options.enabled && options.provider) {
-    const { getProvider } = await resolveCoverageProviderModule(options, loader)
-    return await getProvider()
-  }
+export async function getCoverageProvider(options: CoverageOptions | undefined, loader: Loader): Promise<CoverageProvider | null> {
+  const coverageModule = await resolveCoverageProviderModule(options, loader)
+
+  if (coverageModule)
+    return coverageModule.getProvider()
+
   return null
 }
 
-export async function takeCoverageInsideWorker(options: CoverageOptions, loader: Loader) {
-  if (options.enabled && options.provider) {
-    const { takeCoverage } = await resolveCoverageProviderModule(options, loader)
-    return await takeCoverage?.()
-  }
+export async function startCoverageInsideWorker(options: CoverageOptions | undefined, loader: Loader) {
+  const coverageModule = await resolveCoverageProviderModule(options, loader)
+
+  if (coverageModule)
+    return coverageModule.startCoverage?.()
+
+  return null
+}
+
+export async function takeCoverageInsideWorker(options: CoverageOptions | undefined, loader: Loader) {
+  const coverageModule = await resolveCoverageProviderModule(options, loader)
+
+  if (coverageModule)
+    return coverageModule.takeCoverage?.()
+
+  return null
+}
+
+export async function stopCoverageInsideWorker(options: CoverageOptions | undefined, loader: Loader) {
+  const coverageModule = await resolveCoverageProviderModule(options, loader)
+
+  if (coverageModule)
+    return coverageModule.stopCoverage?.()
+
+  return null
 }

packages/vitest/src/node/pool.ts

@@ -74,8 +74,6 @@ export function createPool(ctx: Vitest): WorkerPool {
     options.minThreads = 1
   }
 
-  ctx.coverageProvider?.onBeforeFilesRun?.()
-
   options.env = {
     TEST: 'true',
     VITEST: 'true',

packages/vitest/src/runtime/entry.ts

@@ -7,8 +7,8 @@ import type { EnvironmentOptions, ResolvedConfig, VitestEnvironment } from '../t
 import { getWorkerState, resetModules } from '../utils'
 import { vi } from '../integrations/vi'
 import { envs } from '../integrations/env'
-import { takeCoverageInsideWorker } from '../integrations/coverage'
 import { distDir } from '../constants'
+import { startCoverageInsideWorker, stopCoverageInsideWorker, takeCoverageInsideWorker } from '../integrations/coverage'
 import { setupGlobalEnv, withEnv } from './setup.node'
 import { rpc } from './rpc'
 import type { VitestExecutor } from './execute'
@@ -79,6 +79,7 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor):
 // browser shouldn't call this!
 export async function run(files: string[], config: ResolvedConfig, executor: VitestExecutor): Promise<void> {
   await setupGlobalEnv(config)
+  await startCoverageInsideWorker(config.coverage, executor)
 
   const workerState = getWorkerState()
 
@@ -159,4 +160,6 @@ export async function run(files: string[], config: ResolvedConfig, executor: Vit
       })
     }
   }
+
+  await stopCoverageInsideWorker(config.coverage, executor)
 }

packages/vitest/src/types/coverage.ts

@@ -10,7 +10,6 @@ export interface CoverageProvider {
   resolveOptions(): ResolvedCoverageOptions
   clean(clean?: boolean): void | Promise<void>
 
-  onBeforeFilesRun?(): void | Promise<void>
   onAfterSuiteRun(meta: AfterSuiteRunMeta): void | Promise<void>
 
   reportCoverage(reportContext?: ReportContext): void | Promise<void>
@@ -32,10 +31,21 @@ export interface CoverageProviderModule {
    * Factory for creating a new coverage provider
    */
   getProvider(): CoverageProvider | Promise<CoverageProvider>
+
+  /**
+   * Executed before tests are run in the worker thread.
+   */
+  startCoverage?(): unknown | Promise<unknown>
+
   /**
    * Executed on after each run in the worker thread. Possible to return a payload passed to the provider
    */
   takeCoverage?(): unknown | Promise<unknown>
+
+  /**
+   * Executed after all tests have been run in the worker thread.
+   */
+  stopCoverage?(): unknown | Promise<unknown>
 }
 
 export type CoverageReporter =

test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap

@@ -6,7 +6,6 @@ exports[`custom json report 1`] = `
     "initialized with context",
     "resolveOptions",
     "clean with force",
-    "onBeforeFilesRun",
     "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}",
     "reportCoverage with {\\"allTestsRun\\":true}",
   ],

test/coverage-test/custom-provider.ts

@@ -9,8 +9,30 @@ const CustomCoverageProviderModule: CoverageProviderModule = {
   },
 
   takeCoverage() {
+    // @ts-expect-error -- untyped
+    globalThis.CUSTOM_PROVIDER_TAKE_COVERAGE = true
+
+    // @ts-expect-error -- untyped
+    if (!globalThis.CUSTOM_PROVIDER_START_COVERAGE)
+      throw new Error('takeCoverage was called before startCoverage!')
+
     return { customCoverage: 'Coverage report passed from workers to main thread' }
   },
+
+  startCoverage() {
+    // @ts-expect-error -- untyped
+    globalThis.CUSTOM_PROVIDER_START_COVERAGE = true
+  },
+
+  stopCoverage() {
+    // @ts-expect-error -- untyped
+    if (!globalThis.CUSTOM_PROVIDER_START_COVERAGE)
+      throw new Error('stopCoverage was called before startCoverage!')
+
+    // @ts-expect-error -- untyped
+    if (!globalThis.CUSTOM_PROVIDER_TAKE_COVERAGE)
+      throw new Error('stopCoverage was called before takeCoverage!')
+  },
 }
 
 /**
@@ -33,10 +55,6 @@ class CustomCoverageProvider implements CoverageProvider {
     this.calls.add(`clean ${force ? 'with' : 'without'} force`)
   }
 
-  onBeforeFilesRun() {
-    this.calls.add('onBeforeFilesRun')
-  }
-
   onAfterSuiteRun(meta: AfterSuiteRunMeta) {
     this.calls.add(`onAfterSuiteRun with ${JSON.stringify(meta)}`)
   }

test/coverage-test/test/configuration-options.test-d.ts

@@ -116,6 +116,8 @@ test('provider module', () => {
       }
     },
     takeCoverage() {},
+    startCoverage() {},
+    stopCoverage() {},
   })
 })