From e1652163564fef635a17d3e449de2e3bd9cc1818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ari=20Perkki=C3=B6?= Date: Sat, 25 Feb 2023 10:02:36 +0200 Subject: [PATCH] feat(coverage): automatic threshold updating (#2886) Closes #1241 --- docs/config/index.md | 10 +++ packages/coverage-c8/rollup.config.js | 1 + packages/coverage-c8/src/provider.ts | 84 ++++++++----------- packages/coverage-istanbul/rollup.config.js | 1 + packages/coverage-istanbul/src/provider.ts | 69 +++++++-------- packages/vitest/coverage.d.ts | 1 + packages/vitest/package.json | 5 ++ packages/vitest/rollup.config.js | 2 + packages/vitest/src/coverage.ts | 1 + packages/vitest/src/types/coverage.ts | 7 ++ packages/vitest/src/utils/coverage.ts | 82 ++++++++++++++++++ pnpm-lock.yaml | 2 + .../__snapshots__/c8.report.test.ts.snap | 2 +- .../__snapshots__/custom.report.test.ts.snap | 2 +- .../istanbul.report.test.ts.snap | 2 +- .../generic.report.test.ts | 17 ++++ .../test/__snapshots__/vue.test.ts.snap | 2 +- test/coverage-test/vitest.config.ts | 7 ++ tsconfig.json | 1 + 19 files changed, 203 insertions(+), 95 deletions(-) create mode 100644 packages/vitest/coverage.d.ts create mode 100644 packages/vitest/src/coverage.ts create mode 100644 packages/vitest/src/utils/coverage.ts diff --git a/docs/config/index.md b/docs/config/index.md index 0e5abfaf1d84..79d4dee7ae5f 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -789,6 +789,16 @@ Do not show files with 100% statement, branch, and function coverage. Check thresholds per file. See `lines`, `functions`, `branches` and `statements` for the actual thresholds. +#### thresholdAutoUpdate + +- **Type:** `boolean` +- **Default:** `false` +- **Available for providers:** `'c8' | 'istanbul'` +- **CLI:** `--coverage.thresholdAutoUpdate=` + +Update threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. +This option helps to maintain thresholds when coverage is improved. + #### lines - **Type:** `number` diff --git a/packages/coverage-c8/rollup.config.js b/packages/coverage-c8/rollup.config.js index 50cfe07d43f2..cbee35392bc8 100644 --- a/packages/coverage-c8/rollup.config.js +++ b/packages/coverage-c8/rollup.config.js @@ -21,6 +21,7 @@ const external = [ 'vitest', 'vitest/node', 'vitest/config', + 'vitest/coverage', ] const plugins = [ diff --git a/packages/coverage-c8/src/provider.ts b/packages/coverage-c8/src/provider.ts index 545fd304d2a3..0d5c419e9c0c 100644 --- a/packages/coverage-c8/src/provider.ts +++ b/packages/coverage-c8/src/provider.ts @@ -6,6 +6,7 @@ import c from 'picocolors' import { provider } from 'std-env' import type { EncodedSourceMap } from 'vite-node' import { coverageConfigDefaults } from 'vitest/config' +import { BaseCoverageProvider } from 'vitest/coverage' // eslint-disable-next-line no-restricted-imports import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest' import type { Vitest } from 'vitest/node' @@ -17,7 +18,7 @@ import { checkCoverages } from 'c8/lib/commands/check-coverage.js' type Options = ResolvedCoverageOptions<'c8'> -export class C8CoverageProvider implements CoverageProvider { +export class C8CoverageProvider extends BaseCoverageProvider implements CoverageProvider { name = 'c8' ctx!: Vitest @@ -25,8 +26,28 @@ export class C8CoverageProvider implements CoverageProvider { coverages: Profiler.TakePreciseCoverageReturnType[] = [] initialize(ctx: Vitest) { + const config: CoverageC8Options = ctx.config.coverage + this.ctx = ctx - this.options = resolveC8Options(ctx.config.coverage, ctx.config.root) + this.options = { + ...coverageConfigDefaults, + + // Provider specific defaults + excludeNodeModules: true, + allowExternal: false, + + // User's options + ...config, + + // Resolved fields + provider: 'c8', + reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter), + reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory), + lines: config['100'] ? 100 : config.lines, + functions: config['100'] ? 100 : config.functions, + branches: config['100'] ? 100 : config.branches, + statements: config['100'] ? 100 : config.statements, + } } resolveOptions() { @@ -156,55 +177,18 @@ export class C8CoverageProvider implements CoverageProvider { await report.run() await checkCoverages(options, report) - } -} - -function resolveC8Options(options: CoverageC8Options, root: string): Options { - const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory) - - const resolved: Options = { - ...coverageConfigDefaults, - - // Provider specific defaults - excludeNodeModules: true, - allowExternal: false, - - // User's options - ...options, - - // Resolved fields - provider: 'c8', - reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter), - reportsDirectory, - } - if (options['100']) { - resolved.lines = 100 - resolved.functions = 100 - resolved.branches = 100 - resolved.statements = 100 - } - - return resolved -} - -function resolveReporters(configReporters: NonNullable): Options['reporter'] { - // E.g. { reporter: "html" } - if (!Array.isArray(configReporters)) - return [[configReporters, {}]] - - const resolvedReporters: Options['reporter'] = [] - - for (const reporter of configReporters) { - if (Array.isArray(reporter)) { - // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]} - resolvedReporters.push([reporter[0], reporter[1] || {}]) - } - else { - // E.g. { reporter: ["html", "json"]} - resolvedReporters.push([reporter, {}]) + if (this.options.thresholdAutoUpdate && allTestsRun) { + this.updateThresholds({ + coverageMap: await report.getCoverageMapFromAllCoverageFiles(), + thresholds: { + branches: this.options.branches, + functions: this.options.functions, + lines: this.options.lines, + statements: this.options.statements, + }, + configurationFile: this.ctx.server.config.configFile, + }) } } - - return resolvedReporters } diff --git a/packages/coverage-istanbul/rollup.config.js b/packages/coverage-istanbul/rollup.config.js index 66fbbd080629..0372998f1117 100644 --- a/packages/coverage-istanbul/rollup.config.js +++ b/packages/coverage-istanbul/rollup.config.js @@ -19,6 +19,7 @@ const external = [ 'vitest', 'vitest/node', 'vitest/config', + 'vitest/coverage', ] const plugins = [ diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index 5abc0058a52b..f67d97daa96c 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -4,6 +4,7 @@ import { relative, resolve } from 'pathe' import type { TransformPluginContext } from 'rollup' import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest' import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config' +import { BaseCoverageProvider } from 'vitest/coverage' import libReport from 'istanbul-lib-report' import reports from 'istanbul-reports' import type { CoverageMap } from 'istanbul-lib-coverage' @@ -31,7 +32,7 @@ interface TestExclude { } } -export class IstanbulCoverageProvider implements CoverageProvider { +export class IstanbulCoverageProvider extends BaseCoverageProvider implements CoverageProvider { name = 'istanbul' ctx!: Vitest @@ -48,8 +49,20 @@ export class IstanbulCoverageProvider implements CoverageProvider { coverages: any[] = [] initialize(ctx: Vitest) { + const config: CoverageIstanbulOptions = ctx.config.coverage + this.ctx = ctx - this.options = resolveIstanbulOptions(ctx.config.coverage, ctx.config.root) + this.options = { + ...coverageConfigDefaults, + + // User's options + ...config, + + // Resolved fields + provider: 'istanbul', + reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory), + reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter), + } this.instrumenter = createInstrumenter({ produceSourceMap: true, @@ -141,6 +154,19 @@ export class IstanbulCoverageProvider implements CoverageProvider { statements: this.options.statements, }) } + + if (this.options.thresholdAutoUpdate && allTestsRun) { + this.updateThresholds({ + coverageMap, + thresholds: { + branches: this.options.branches, + functions: this.options.functions, + lines: this.options.lines, + statements: this.options.statements, + }, + configurationFile: this.ctx.server.config.configFile, + }) + } } checkThresholds(coverageMap: CoverageMap, thresholds: Record) { @@ -220,24 +246,6 @@ export class IstanbulCoverageProvider implements CoverageProvider { } } -function resolveIstanbulOptions(options: CoverageIstanbulOptions, root: string): Options { - const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory) - - const resolved: Options = { - ...coverageConfigDefaults, - - // User's options - ...options, - - // Resolved fields - provider: 'istanbul', - reportsDirectory, - reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter), - } - - return resolved -} - /** * Remove possible query parameters from filenames * - From `/src/components/Header.component.ts?vue&type=script&src=true&lang.ts` @@ -287,24 +295,3 @@ function isEmptyCoverageRange(range: libCoverage.Range) { || range.end.column === undefined ) } - -function resolveReporters(configReporters: NonNullable): Options['reporter'] { - // E.g. { reporter: "html" } - if (!Array.isArray(configReporters)) - return [[configReporters, {}]] - - const resolvedReporters: Options['reporter'] = [] - - for (const reporter of configReporters) { - if (Array.isArray(reporter)) { - // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]} - resolvedReporters.push([reporter[0], reporter[1] || {}]) - } - else { - // E.g. { reporter: ["html", "json"]} - resolvedReporters.push([reporter, {}]) - } - } - - return resolvedReporters -} diff --git a/packages/vitest/coverage.d.ts b/packages/vitest/coverage.d.ts new file mode 100644 index 000000000000..cf1145ffa2f6 --- /dev/null +++ b/packages/vitest/coverage.d.ts @@ -0,0 +1 @@ +export * from './dist/coverage.js' diff --git a/packages/vitest/package.json b/packages/vitest/package.json index 0af61735b1f7..13c7686f133f 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -65,6 +65,10 @@ "types": "./config.d.ts", "require": "./dist/config.cjs", "import": "./dist/config.js" + }, + "./coverage": { + "types": "./coverage.d.ts", + "import": "./dist/coverage.js" } }, "main": "./dist/index.js", @@ -144,6 +148,7 @@ "@edge-runtime/vm": "2.0.2", "@sinonjs/fake-timers": "^10.0.2", "@types/diff": "^5.0.2", + "@types/istanbul-lib-coverage": "^2.0.4", "@types/istanbul-reports": "^3.0.1", "@types/jsdom": "^21.1.0", "@types/micromatch": "^4.0.2", diff --git a/packages/vitest/rollup.config.js b/packages/vitest/rollup.config.js index eb650f92df6e..d803b1ffa218 100644 --- a/packages/vitest/rollup.config.js +++ b/packages/vitest/rollup.config.js @@ -26,6 +26,7 @@ const entries = [ 'src/runtime/loader.ts', 'src/runtime/entry.ts', 'src/integrations/spy.ts', + 'src/coverage.ts', ] const dtsEntries = [ @@ -36,6 +37,7 @@ const dtsEntries = [ 'src/runners.ts', 'src/suite.ts', 'src/config.ts', + 'src/coverage.ts', ] const external = [ diff --git a/packages/vitest/src/coverage.ts b/packages/vitest/src/coverage.ts new file mode 100644 index 000000000000..5dccd7b1fd9e --- /dev/null +++ b/packages/vitest/src/coverage.ts @@ -0,0 +1 @@ +export { BaseCoverageProvider } from './utils/coverage' diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index efa648a82667..faa09bf62f49 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -186,6 +186,13 @@ export interface BaseCoverageOptions { * @default undefined */ statements?: number + + /** + * Update threshold values automatically when current coverage is higher than earlier thresholds + * + * @default false + */ + thresholdAutoUpdate?: boolean } export interface CoverageIstanbulOptions extends BaseCoverageOptions { diff --git a/packages/vitest/src/utils/coverage.ts b/packages/vitest/src/utils/coverage.ts new file mode 100644 index 000000000000..7e109a7d76f6 --- /dev/null +++ b/packages/vitest/src/utils/coverage.ts @@ -0,0 +1,82 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import type { CoverageMap } from 'istanbul-lib-coverage' +import type { BaseCoverageOptions, ResolvedCoverageOptions } from '../types' + +type Threshold = 'lines' | 'functions' | 'statements' | 'branches' + +const THRESHOLD_KEYS: Readonly = ['lines', 'functions', 'statements', 'branches'] + +export class BaseCoverageProvider { + /** + * Check if current coverage is above configured thresholds and bump the thresholds if needed + */ + updateThresholds({ configurationFile, coverageMap, thresholds }: { + coverageMap: CoverageMap + thresholds: Record + configurationFile?: string + }) { + // Thresholds cannot be updated if there is no configuration file and + // feature was enabled by CLI, e.g. --coverage.thresholdAutoUpdate + if (!configurationFile) + throw new Error('Missing configurationFile. The "coverage.thresholdAutoUpdate" can only be enabled when configuration file is used.') + + const summary = coverageMap.getCoverageSummary() + const thresholdsToUpdate: Threshold[] = [] + + for (const key of THRESHOLD_KEYS) { + const threshold = thresholds[key] || 100 + const actual = summary[key].pct + + if (actual > threshold) + thresholdsToUpdate.push(key) + } + + if (thresholdsToUpdate.length === 0) + return + + const originalConfig = readFileSync(configurationFile, 'utf8') + let updatedConfig = originalConfig + + for (const threshold of thresholdsToUpdate) { + // Find the exact match from the configuration file and replace the value + const previousThreshold = (thresholds[threshold] || 100).toString() + const pattern = new RegExp(`(${threshold}\\s*:\\s*)${previousThreshold.replace('.', '\\.')}`) + const matches = originalConfig.match(pattern) + + if (matches) + updatedConfig = updatedConfig.replace(matches[0], matches[1] + summary[threshold].pct) + else + console.error(`Unable to update coverage threshold ${threshold}. No threshold found using pattern ${pattern}`) + } + + if (updatedConfig !== originalConfig) { + // eslint-disable-next-line no-console + console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.') + writeFileSync(configurationFile, updatedConfig, 'utf-8') + } + } + + /** + * Resolve reporters from various configuration options + */ + resolveReporters(configReporters: NonNullable): ResolvedCoverageOptions['reporter'] { + // E.g. { reporter: "html" } + if (!Array.isArray(configReporters)) + return [[configReporters, {}]] + + const resolvedReporters: ResolvedCoverageOptions['reporter'] = [] + + for (const reporter of configReporters) { + if (Array.isArray(reporter)) { + // E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]} + resolvedReporters.push([reporter[0], reporter[1] || {}]) + } + else { + // E.g. { reporter: ["html", "json"]} + resolvedReporters.push([reporter, {}]) + } + } + + return resolvedReporters + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f6247897dfa..3187cd0cce72 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -844,6 +844,7 @@ importers: '@types/chai': ^4.3.4 '@types/chai-subset': ^1.3.3 '@types/diff': ^5.0.2 + '@types/istanbul-lib-coverage': ^2.0.4 '@types/istanbul-reports': ^3.0.1 '@types/jsdom': ^21.1.0 '@types/micromatch': ^4.0.2 @@ -928,6 +929,7 @@ importers: '@edge-runtime/vm': 2.0.2 '@sinonjs/fake-timers': 10.0.2 '@types/diff': 5.0.2 + '@types/istanbul-lib-coverage': 2.0.4 '@types/istanbul-reports': 3.0.1 '@types/jsdom': 21.1.0 '@types/micromatch': 4.0.2 diff --git a/test/coverage-test/coverage-report-tests/__snapshots__/c8.report.test.ts.snap b/test/coverage-test/coverage-report-tests/__snapshots__/c8.report.test.ts.snap index 4dfc9adb9146..bc105adcc360 100644 --- a/test/coverage-test/coverage-report-tests/__snapshots__/c8.report.test.ts.snap +++ b/test/coverage-test/coverage-report-tests/__snapshots__/c8.report.test.ts.snap @@ -1,4 +1,4 @@ -// Vitest Snapshot v1 +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`c8 json report 1`] = ` { diff --git a/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap b/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap index 1bb36a68bfef..c3ea52c099cb 100644 --- a/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap +++ b/test/coverage-test/coverage-report-tests/__snapshots__/custom.report.test.ts.snap @@ -1,4 +1,4 @@ -// Vitest Snapshot v1 +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`custom json report 1`] = ` { diff --git a/test/coverage-test/coverage-report-tests/__snapshots__/istanbul.report.test.ts.snap b/test/coverage-test/coverage-report-tests/__snapshots__/istanbul.report.test.ts.snap index 4894bfa52465..4bdaff5fe8e9 100644 --- a/test/coverage-test/coverage-report-tests/__snapshots__/istanbul.report.test.ts.snap +++ b/test/coverage-test/coverage-report-tests/__snapshots__/istanbul.report.test.ts.snap @@ -1,4 +1,4 @@ -// Vitest Snapshot v1 +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`istanbul json report 1`] = ` { diff --git a/test/coverage-test/coverage-report-tests/generic.report.test.ts b/test/coverage-test/coverage-report-tests/generic.report.test.ts index f79271bc7575..4262250041fa 100644 --- a/test/coverage-test/coverage-report-tests/generic.report.test.ts +++ b/test/coverage-test/coverage-report-tests/generic.report.test.ts @@ -63,3 +63,20 @@ test('files should not contain a setup file', () => { expect(srcFiles).not.toContain('another-setup.ts.html') }) + +test('thresholdAutoUpdate updates thresholds', async () => { + const configFilename = resolve('./vitest.config.ts') + const configContents = fs.readFileSync(configFilename, 'utf-8') + + for (const threshold of ['functions', 'branches', 'lines', 'statements']) { + const match = configContents.match(new RegExp(`${threshold}: (?[\\d|\\.]+)`)) + const coverage = match?.groups?.coverage || '0' + + // Configuration has fixed value of 1.01 set for each threshold + expect(parseInt(coverage)).toBeGreaterThan(1.01) + } + + // Update thresholds back to fixed values + const updatedConfig = configContents.replace(/(branches|functions|lines|statements): ([\d|\.])+/g, '$1: 1.01') + fs.writeFileSync(configFilename, updatedConfig) +}) diff --git a/test/coverage-test/test/__snapshots__/vue.test.ts.snap b/test/coverage-test/test/__snapshots__/vue.test.ts.snap index 703946be1786..ddef165587e7 100644 --- a/test/coverage-test/test/__snapshots__/vue.test.ts.snap +++ b/test/coverage-test/test/__snapshots__/vue.test.ts.snap @@ -1,4 +1,4 @@ -// Vitest Snapshot v1 +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`vue 3 coverage 1`] = ` "
4 x 2 = 8
diff --git a/test/coverage-test/vitest.config.ts b/test/coverage-test/vitest.config.ts index 559f7e9e34e1..0eb103425292 100644 --- a/test/coverage-test/vitest.config.ts +++ b/test/coverage-test/vitest.config.ts @@ -25,6 +25,13 @@ export default defineConfig({ ['lcov', {}], ['json', { file: 'custom-json-report-name.json' }], ], + + // These will be updated by tests and reseted back by generic.report.test.ts + thresholdAutoUpdate: true, + functions: 1.01, + branches: 1.01, + lines: 1.01, + statements: 1.01, }, setupFiles: [ resolve(__dirname, './setup.ts'), diff --git a/tsconfig.json b/tsconfig.json index 54fbe9d3a339..aa7534d66b93 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,7 @@ "vitest/globals": ["./packages/vitest/globals.d.ts"], "vitest/node": ["./packages/vitest/src/node/index.ts"], "vitest/config": ["./packages/vitest/src/config.ts"], + "vitest/coverage": ["./packages/vitest/src/coverage.ts"], "vitest/browser": ["./packages/vitest/src/browser.ts"], "vitest/runners": ["./packages/vitest/src/runners.ts"], "vite-node": ["./packages/vite-node/src/index.ts"],