From 81604bcee06cb64770a1ffcfaefa4718fe380eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ari=20Perkki=C3=B6?= Date: Mon, 13 Feb 2023 14:28:52 +0200 Subject: [PATCH] fix(coverage): custom providers to work inside worker threads (#2817) --- docs/config/index.md | 12 ++- docs/guide/coverage.md | 34 +++++++- packages/vitest/src/integrations/coverage.ts | 41 +++++---- packages/vitest/src/node/cli-api.ts | 4 +- packages/vitest/src/node/core.ts | 2 +- packages/vitest/src/runtime/entry.ts | 2 +- packages/vitest/src/types/coverage.ts | 14 +++- .../__snapshots__/custom.report.test.ts.snap | 26 ++++++ .../custom.report.test.ts | 12 +++ .../coverage-report-tests/utils.ts | 6 +- test/coverage-test/custom-provider.ts | 75 +++++++++++++++++ test/coverage-test/package.json | 5 +- .../test/configuration-options.test-d.ts | 84 ++++++++++++------- test/coverage-test/testing.mjs | 52 +++--------- test/coverage-test/vitest.config.ts | 8 +- 15 files changed, 274 insertions(+), 103 deletions(-) create mode 100644 test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap create mode 100644 test/coverage-test/coverage-report-tests/custom.report.test.ts create mode 100644 test/coverage-test/custom-provider.ts diff --git a/docs/config/index.md b/docs/config/index.md index 679dfdf35553..345812580864 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -617,7 +617,7 @@ Isolate environment for each test file. Does not work if you disable [`--threads ### coverage -You can use [`c8`](https://github.com/bcoe/c8) or [`istanbul`](https://istanbul.js.org/) for coverage collection. +You can use [`c8`](https://github.com/bcoe/c8), [`istanbul`](https://istanbul.js.org/) or [a custom coverage solution](/guide/coverage#custom-coverage-provider) for coverage collection. You can provide coverage options to CLI with dot notation: @@ -631,7 +631,7 @@ If you are using coverage options with dot notation, don't forget to specify `-- #### provider -- **Type:** `'c8' | 'istanbul'` +- **Type:** `'c8' | 'istanbul' | 'custom'` - **Default:** `'c8'` - **CLI:** `--coverage.provider=` @@ -863,6 +863,14 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#ignoring-methods) Watermarks for statements, lines, branches and functions. See [istanbul documentation](https://github.com/istanbuljs/nyc#high-and-low-watermarks) for more information. +#### customProviderModule + +- **Type:** `string` +- **Available for providers:** `'custom'` +- **CLI:** `--coverage.customProviderModule=` + +Specifies the module name or path for the custom coverage provider module. See [Guide - Custom Coverage Provider](/guide/coverage#custom-coverage-provider) for more information. + ### testNamePattern - **Type** `string | RegExp` diff --git a/docs/guide/coverage.md b/docs/guide/coverage.md index 2bab6b8855f2..031fcf2e9338 100644 --- a/docs/guide/coverage.md +++ b/docs/guide/coverage.md @@ -71,22 +71,50 @@ export default defineConfig({ ## Custom Coverage Provider -It's also possible to provide your custom coverage provider by passing an object to the `test.coverage.provider`: +It's also possible to provide your custom coverage provider by passing `'custom'` in `test.coverage.provider`: ```ts // vite.config.ts import { defineConfig } from 'vitest/config' -import CustomCoverageProvider from 'my-custom-coverage-provider' export default defineConfig({ test: { coverage: { - provider: CustomCoverageProvider() + provider: 'custom', + customProviderModule: 'my-custom-coverage-provider' }, }, }) ``` +The custom providers require a `customProviderModule` option which is a module name or path where to load the `CoverageProviderModule` from. It must export an object that implements `CoverageProviderModule` as default export: + +```ts +// my-custom-coverage-provider.ts +import type { CoverageProvider, CoverageProviderModule, ResolvedCoverageOptions, Vitest } from 'vitest' + +const CustomCoverageProviderModule: CoverageProviderModule = { + getProvider(): CoverageProvider { + return new CustomCoverageProvider() + }, + + // Implements rest of the CoverageProviderModule ... +} + +class CustomCoverageProvider implements CoverageProvider { + name = 'custom-coverage-provider' + options!: ResolvedCoverageOptions + + initialize(ctx: Vitest) { + this.options = ctx.config.coverage + } + + // Implements rest of the CoverageProvider ... +} + +export default CustomCoverageProviderModule +``` + Please refer to the type definition for more details. ## Changing the default coverage folder location diff --git a/packages/vitest/src/integrations/coverage.ts b/packages/vitest/src/integrations/coverage.ts index a14b4d5dc9f9..e75106eba22f 100644 --- a/packages/vitest/src/integrations/coverage.ts +++ b/packages/vitest/src/integrations/coverage.ts @@ -1,34 +1,47 @@ import { importModule } from 'local-pkg' import type { CoverageOptions, CoverageProvider, CoverageProviderModule } from '../types' -export const CoverageProviderMap = { +interface Loader { + executeId: (id: string) => Promise<{ default: CoverageProviderModule }> +} + +export const CoverageProviderMap: Record = { c8: '@vitest/coverage-c8', istanbul: '@vitest/coverage-istanbul', } -export async function resolveCoverageProvider(provider: NonNullable) { - if (typeof provider === 'string') { - const pkg = CoverageProviderMap[provider] - if (!pkg) - throw new Error(`Unknown coverage provider: ${provider}`) - return await importModule(pkg) +async function resolveCoverageProviderModule(options: CoverageOptions & Required>, loader: Loader) { + const provider = options.provider + + if (provider === 'c8' || provider === 'istanbul') + return await importModule(CoverageProviderMap[provider]) + + let customProviderModule + + try { + customProviderModule = await loader.executeId(options.customProviderModule) } - else { - return provider + catch (error) { + throw new Error(`Failed to load custom CoverageProviderModule from ${options.customProviderModule}`, { cause: error }) } + + if (customProviderModule.default == null) + throw new Error(`Custom CoverageProviderModule loaded from ${options.customProviderModule} was not the default export`) + + return customProviderModule.default } -export async function getCoverageProvider(options?: CoverageOptions): Promise { - if (options?.enabled && options?.provider) { - const { getProvider } = await resolveCoverageProvider(options.provider) +export async function getCoverageProvider(options: CoverageOptions, loader: Loader): Promise { + if (options.enabled && options.provider) { + const { getProvider } = await resolveCoverageProviderModule(options, loader) return await getProvider() } return null } -export async function takeCoverageInsideWorker(options: CoverageOptions) { +export async function takeCoverageInsideWorker(options: CoverageOptions, loader: Loader) { if (options.enabled && options.provider) { - const { takeCoverage } = await resolveCoverageProvider(options.provider) + const { takeCoverage } = await resolveCoverageProviderModule(options, loader) return await takeCoverage?.() } } diff --git a/packages/vitest/src/node/cli-api.ts b/packages/vitest/src/node/cli-api.ts index 62c245d3b8bc..b891429c638a 100644 --- a/packages/vitest/src/node/cli-api.ts +++ b/packages/vitest/src/node/cli-api.ts @@ -50,9 +50,9 @@ export async function startVitest( if (mode === 'test' && ctx.config.coverage.enabled) { const provider = ctx.config.coverage.provider || 'c8' - if (typeof provider === 'string') { - const requiredPackages = CoverageProviderMap[provider] + const requiredPackages = CoverageProviderMap[provider] + if (requiredPackages) { if (!await ensurePackageInstalled(requiredPackages, root)) { process.exitCode = 1 return ctx diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index 844bc1d268dc..2dd8e1e6f9f2 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -124,7 +124,7 @@ export class Vitest { async initCoverageProvider() { if (this.coverageProvider !== undefined) return - this.coverageProvider = await getCoverageProvider(this.config.coverage) + this.coverageProvider = await getCoverageProvider(this.config.coverage, this.runner) if (this.coverageProvider) { await this.coverageProvider.initialize(this) this.config.coverage = this.coverageProvider.resolveOptions() diff --git a/packages/vitest/src/runtime/entry.ts b/packages/vitest/src/runtime/entry.ts index e36336335b00..427d62c482e4 100644 --- a/packages/vitest/src/runtime/entry.ts +++ b/packages/vitest/src/runtime/entry.ts @@ -68,7 +68,7 @@ async function getTestRunner(config: ResolvedConfig, executor: VitestExecutor): const originalOnAfterRun = testRunner.onAfterRun testRunner.onAfterRun = async (files) => { - const coverage = await takeCoverageInsideWorker(config.coverage) + const coverage = await takeCoverageInsideWorker(config.coverage, executor) rpc().onAfterSuiteRun({ coverage }) await originalOnAfterRun?.call(testRunner, files) } diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index db15d9fd3811..c8e26f79493b 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -53,12 +53,13 @@ export type CoverageReporter = | 'text-summary' | 'text' -type Provider = 'c8' | 'istanbul' | CoverageProviderModule | undefined +type Provider = 'c8' | 'istanbul' | 'custom' | undefined export type CoverageOptions = - T extends CoverageProviderModule ? ({ provider: T } & BaseCoverageOptions) : - T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) : - ({ provider?: T } & CoverageC8Options) + T extends 'istanbul' ? ({ provider: T } & CoverageIstanbulOptions) : + T extends 'c8' ? ({ provider: T } & CoverageC8Options) : + T extends 'custom' ? ({ provider: T } & CustomProviderOptions) : + ({ provider?: T } & (CoverageC8Options)) /** Fields that have default values. Internally these will always be defined. */ type FieldsWithDefaultValues = @@ -233,3 +234,8 @@ export interface CoverageC8Options extends BaseCoverageOptions { */ 100?: boolean } + +export interface CustomProviderOptions extends Pick { + /** Name of the module or path to a file to load the custom provider from */ + customProviderModule: string +} diff --git a/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap b/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap new file mode 100644 index 000000000000..a48348f3cd54 --- /dev/null +++ b/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap @@ -0,0 +1,26 @@ +// Vitest Snapshot v1 + +exports[`custom json report 1`] = ` +{ + "calls": [ + "initialized with context", + "resolveOptions", + "clean with force", + "onBeforeFilesRun", + "onAfterSuiteRun with {\\"coverage\\":{\\"customCoverage\\":\\"Coverage report passed from workers to main thread\\"}}", + "reportCoverage with {\\"allTestsRun\\":true}", + ], + "transformedFiles": [ + "/src/Counter/Counter.component.ts", + "/src/Counter/Counter.vue", + "/src/Counter/index.ts", + "/src/Defined.vue", + "/src/Hello.vue", + "/src/another-setup.ts", + "/src/implicitElse.ts", + "/src/importEnv.ts", + "/src/index.mts", + "/src/utils.ts", + ], +} +`; diff --git a/test/coverage-test/coverage-report-tests/custom.report.test.ts b/test/coverage-test/coverage-report-tests/custom.report.test.ts new file mode 100644 index 000000000000..eb06455660de --- /dev/null +++ b/test/coverage-test/coverage-report-tests/custom.report.test.ts @@ -0,0 +1,12 @@ +/* + * Custom coverage provider specific test cases + */ + +import { readFileSync } from 'fs' +import { expect, test } from 'vitest' + +test('custom json report', async () => { + const report = readFileSync('./coverage/custom-coverage-provider-report.json', 'utf-8') + + expect(JSON.parse(report)).toMatchSnapshot() +}) diff --git a/test/coverage-test/coverage-report-tests/utils.ts b/test/coverage-test/coverage-report-tests/utils.ts index 1f52b86061f8..6cc7600a3178 100644 --- a/test/coverage-test/coverage-report-tests/utils.ts +++ b/test/coverage-test/coverage-report-tests/utils.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'fs' import { normalize } from 'pathe' interface CoverageFinalJson { @@ -17,8 +18,7 @@ interface CoverageFinalJson { * Normalizes paths to keep contents consistent between OS's */ export async function readCoverageJson() { - // @ts-expect-error -- generated file - const { default: jsonReport } = await import('./coverage/coverage-final.json') as CoverageFinalJson + const jsonReport = JSON.parse(readFileSync('./coverage/coverage-final.json', 'utf8')) as CoverageFinalJson const normalizedReport: CoverageFinalJson['default'] = {} @@ -30,6 +30,6 @@ export async function readCoverageJson() { return normalizedReport } -function normalizeFilename(filename: string) { +export function normalizeFilename(filename: string) { return normalize(filename).replace(normalize(process.cwd()), '') } diff --git a/test/coverage-test/custom-provider.ts b/test/coverage-test/custom-provider.ts new file mode 100644 index 000000000000..6109ce9e4bcb --- /dev/null +++ b/test/coverage-test/custom-provider.ts @@ -0,0 +1,75 @@ +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'fs' +import type { AfterSuiteRunMeta, CoverageProvider, CoverageProviderModule, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest' + +import { normalizeFilename } from './coverage-report-tests/utils' + +const CustomCoverageProviderModule: CoverageProviderModule = { + getProvider(): CoverageProvider { + return new CustomCoverageProvider() + }, + + takeCoverage() { + return { customCoverage: 'Coverage report passed from workers to main thread' } + }, +} + +/** + * Provider that simply keeps track of the functions that were called + */ +class CustomCoverageProvider implements CoverageProvider { + name = 'custom-coverage-provider' + + options!: ResolvedCoverageOptions + calls: Set = new Set() + transformedFiles: Set = new Set() + + initialize(ctx: Vitest) { + this.options = ctx.config.coverage + + this.calls.add(`initialized ${ctx ? 'with' : 'without'} context`) + } + + clean(force: boolean) { + this.calls.add(`clean ${force ? 'with' : 'without'} force`) + } + + onBeforeFilesRun() { + this.calls.add('onBeforeFilesRun') + } + + onAfterSuiteRun(meta: AfterSuiteRunMeta) { + this.calls.add(`onAfterSuiteRun with ${JSON.stringify(meta)}`) + } + + reportCoverage(reportContext?: ReportContext) { + this.calls.add(`reportCoverage with ${JSON.stringify(reportContext)}`) + + const jsonReport = JSON.stringify({ + calls: Array.from(this.calls.values()), + transformedFiles: Array.from(this.transformedFiles.values()).sort(), + }, null, 2) + + if (existsSync('./coverage')) + rmSync('./coverage', { maxRetries: 10, recursive: true }) + + mkdirSync('./coverage') + writeFileSync('./coverage/custom-coverage-provider-report.json', jsonReport, 'utf-8') + } + + onFileTransform(code: string, id: string) { + const filename = normalizeFilename(id).split('?')[0] + + if (/\/src\//.test(filename)) + this.transformedFiles.add(filename) + + return { code } + } + + resolveOptions(): ResolvedCoverageOptions { + this.calls.add('resolveOptions') + + return this.options + } +} + +export default CustomCoverageProviderModule diff --git a/test/coverage-test/package.json b/test/coverage-test/package.json index 90fd498cbe73..46d49c7a1a11 100644 --- a/test/coverage-test/package.json +++ b/test/coverage-test/package.json @@ -2,10 +2,11 @@ "name": "@vitest/test-coverage", "private": true, "scripts": { - "test": "pnpm run test:c8 && pnpm run test:istanbul && pnpm run test:types", + "test": "pnpm test:c8 && pnpm test:istanbul && pnpm test:custom && pnpm test:types", "test:c8": "node ./testing.mjs --provider c8", + "test:custom": "node ./testing.mjs --provider custom", "test:istanbul": "node ./testing.mjs --provider istanbul", - "test:types": "vitest typecheck --run" + "test:types": "vitest typecheck --run --reporter verbose" }, "devDependencies": { "@vitejs/plugin-vue": "latest", diff --git a/test/coverage-test/test/configuration-options.test-d.ts b/test/coverage-test/test/configuration-options.test-d.ts index 025ef4155d50..0f0e60cb7be9 100644 --- a/test/coverage-test/test/configuration-options.test-d.ts +++ b/test/coverage-test/test/configuration-options.test-d.ts @@ -1,5 +1,5 @@ import { assertType, test } from 'vitest' -import type { ResolvedCoverageOptions, Vitest } from 'vitest' +import type { CoverageProviderModule, ResolvedCoverageOptions, Vitest } from 'vitest' import type { defineConfig } from 'vitest/config' type NarrowToTestConfig = T extends { test?: any } ? NonNullable : never @@ -10,39 +10,14 @@ test('providers, built-in', () => { assertType({ provider: 'c8' }) assertType({ provider: 'istanbul' }) - // @ts-expect-error -- String options must be known built-in's - assertType({ provider: 'unknown-reporter' }) + // @ts-expect-error -- String options must be known ones only + assertType({ provider: 'unknown-provider' }) }) test('providers, custom', () => { assertType({ - provider: { - getProvider() { - return { - name: 'custom-provider', - initialize(_: Vitest) {}, - resolveOptions(): ResolvedCoverageOptions { - return { - clean: true, - cleanOnRerun: true, - enabled: true, - exclude: ['string'], - extension: ['string'], - reporter: ['html', 'json'], - reportsDirectory: 'string', - } - }, - clean(_: boolean) {}, - onBeforeFilesRun() {}, - onAfterSuiteRun({ coverage: _coverage }) {}, - reportCoverage() {}, - onFileTransform(_code: string, _id: string, ctx) { - ctx.getCombinedSourcemap() - }, - } - }, - takeCoverage() {}, - }, + provider: 'custom', + customProviderModule: 'custom-provider-module.ts', }) }) @@ -95,6 +70,55 @@ test('provider specific options, istanbul', () => { }) }) +test('provider specific options, custom', () => { + assertType({ + provider: 'custom', + customProviderModule: 'custom-provider-module.ts', + enabled: true, + }) + + // @ts-expect-error -- customProviderModule is required + assertType({ provider: 'custom' }) + + assertType({ + provider: 'custom', + customProviderModule: 'some-module', + + // @ts-expect-error -- typings of BaseCoverageOptions still apply + enabled: 'not boolean', + }) +}) + +test('provider module', () => { + assertType({ + getProvider() { + return { + name: 'custom-provider', + initialize(_: Vitest) {}, + resolveOptions(): ResolvedCoverageOptions { + return { + clean: true, + cleanOnRerun: true, + enabled: true, + exclude: ['string'], + extension: ['string'], + reporter: ['html', 'json'], + reportsDirectory: 'string', + } + }, + clean(_: boolean) {}, + onBeforeFilesRun() {}, + onAfterSuiteRun({ coverage: _coverage }) {}, + reportCoverage() {}, + onFileTransform(_code: string, _id: string, ctx) { + ctx.getCombinedSourcemap() + }, + } + }, + takeCoverage() {}, + }) +}) + test('reporters, single', () => { assertType({ reporter: 'clover' }) assertType({ reporter: 'cobertura' }) diff --git a/test/coverage-test/testing.mjs b/test/coverage-test/testing.mjs index f419f1acbcab..9113b3c7fa60 100644 --- a/test/coverage-test/testing.mjs +++ b/test/coverage-test/testing.mjs @@ -3,61 +3,35 @@ import { startVitest } from 'vitest/node' // Set this to true when intentionally updating the snapshots const UPDATE_SNAPSHOTS = false -const provider = getArgument('--provider') +const provider = process.argv[1 + process.argv.indexOf('--provider')] const configs = [ // Run test cases. Generates coverage report. ['test/', { include: ['test/*.test.*'], exclude: ['coverage-report-tests/**/*'], + coverage: { enabled: true }, }], // Run tests for checking coverage report contents. ['coverage-report-tests', { include: [ - './coverage-report-tests/generic.report.test.ts', + ['c8', 'istanbul'].includes(provider) && './coverage-report-tests/generic.report.test.ts', `./coverage-report-tests/${provider}.report.test.ts`, - ], + ].filter(Boolean), coverage: { enabled: false, clean: false }, }], ] -runTests() +for (const threads of [true, false]) { + for (const [directory, config] of configs) { + await startVitest('test', [directory], { + ...config, + update: UPDATE_SNAPSHOTS, + threads, + }) -async function runTests() { - for (const threads of [true, false]) { - for (const [directory, config] of configs) { - await startVitest('test', [directory], { - run: true, - update: UPDATE_SNAPSHOTS, - ...config, - threads, - coverage: { - include: ['src/**'], - provider, - ...config.coverage, - }, - }) - - if (process.exitCode) - process.exit() - } + if (process.exitCode) + process.exit() } - - process.exit(0) -} - -function getArgument(name) { - const args = process.argv - const index = args.indexOf(name) - - if (index === -1) - throw new Error(`Missing argument ${name}, received ${args}`) - - const value = args[index + 1] - - if (!value) - throw new Error(`Missing value of ${name}, received ${args}`) - - return value } diff --git a/test/coverage-test/vitest.config.ts b/test/coverage-test/vitest.config.ts index 04ecdaa76505..325f715f4112 100644 --- a/test/coverage-test/vitest.config.ts +++ b/test/coverage-test/vitest.config.ts @@ -2,6 +2,8 @@ import { resolve } from 'pathe' import { defineConfig } from 'vitest/config' import vue from '@vitejs/plugin-vue' +const provider = process.argv[1 + process.argv.indexOf('--provider')] + export default defineConfig({ plugins: [ vue(), @@ -10,9 +12,11 @@ export default defineConfig({ MY_CONSTANT: '"my constant"', }, test: { - reporters: 'verbose', + watch: false, coverage: { - enabled: true, + provider: provider as any, + customProviderModule: provider === 'custom' ? 'custom-provider' : undefined, + include: ['src/**'], clean: true, all: true, reporter: ['html', 'text', 'lcov', 'json'],