From 4953410e8d96bf95f9d92bb2c8413f25b5b371b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ari=20Perkki=C3=B6?= Date: Fri, 17 Nov 2023 20:24:04 +0200 Subject: [PATCH] feat!(coverage): glob based coverage thresholds (#4442) --- docs/config/index.md | 101 ++++--- packages/coverage-istanbul/package.json | 1 + packages/coverage-istanbul/src/provider.ts | 65 +++-- packages/coverage-v8/package.json | 1 + packages/coverage-v8/src/provider.ts | 65 +++-- packages/vitest/src/node/cli.ts | 2 +- packages/vitest/src/types/coverage.ts | 88 +++---- packages/vitest/src/utils/coverage.ts | 249 ++++++++++++------ pnpm-lock.yaml | 16 ++ test/config/test/failures.test.ts | 23 ++ test/config/vitest.config.ts | 1 + .../__snapshots__/v8.report.test.ts.snap | 8 +- .../generic.report.test.ts | 46 +++- .../option-tests/threshold-100.test.ts | 18 ++ test/coverage-test/package.json | 1 + .../test/configuration-options.test-d.ts | 26 +- test/coverage-test/testing-options.mjs | 15 ++ test/coverage-test/vitest.config.ts | 20 +- 18 files changed, 510 insertions(+), 236 deletions(-) create mode 100644 test/coverage-test/option-tests/threshold-100.test.ts diff --git a/docs/config/index.md b/docs/config/index.md index 4ea03b85a972..34375813e7f1 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -1181,70 +1181,109 @@ Collect coverage of files outside the [project `root`](https://vitest.dev/config Do not show files with 100% statement, branch, and function coverage. -#### coverage.perFile +#### coverage.thresholds -- **Type:** `boolean` -- **Default:** `false` -- **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.perFile`, `--coverage.perFile=false` - -Check thresholds per file. -See `lines`, `functions`, `branches` and `statements` for the actual thresholds. +Options for coverage thresholds -#### coverage.thresholdAutoUpdate +##### coverage.thresholds.lines -- **Type:** `boolean` -- **Default:** `false` +- **Type:** `number` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.thresholdAutoUpdate=` +- **CLI:** `--coverage.thresholds.lines=` -Update threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. -This option helps to maintain thresholds when coverage is improved. +Global threshold for lines. +See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. -#### coverage.lines +##### coverage.thresholds.functions - **Type:** `number` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.lines=` +- **CLI:** `--coverage.thresholds.functions=` -Threshold for lines. +Global threshold for functions. See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. -#### coverage.functions +##### coverage.thresholds.branches - **Type:** `number` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.functions=` +- **CLI:** `--coverage.thresholds.branches=` -Threshold for functions. +Global threshold for branches. See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. -#### coverage.branches +##### coverage.thresholds.statements - **Type:** `number` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.branches=` +- **CLI:** `--coverage.thresholds.statements=` -Threshold for branches. +Global threshold for statements. See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. -#### coverage.statements +##### coverage.thresholds.perFile -- **Type:** `number` +- **Type:** `boolean` +- **Default:** `false` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.statements=` +- **CLI:** `--coverage.thresholds.perFile`, `--coverage.thresholds.perFile=false` -Threshold for statements. -See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information. +Check thresholds per file. -#### coverage.100 +##### coverage.thresholds.autoUpdate - **Type:** `boolean` - **Default:** `false` - **Available for providers:** `'v8' | 'istanbul'` -- **CLI:** `--coverage.100`, `--coverage.100=false` +- **CLI:** `--coverage.thresholds.autoUpdate=` + +Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. +This option helps to maintain thresholds when coverage is improved. + +##### coverage.thresholds.100 -Shortcut for `--coverage.lines 100 --coverage.functions 100 --coverage.branches 100 --coverage.statements 100`. +- **Type:** `boolean` +- **Default:** `false` +- **Available for providers:** `'v8' | 'istanbul'` +- **CLI:** `--coverage.thresholds.100`, `--coverage.thresholds.100=false` + +Sets global thresholds to 100. +Shortcut for `--coverage.thresholds.lines 100 --coverage.thresholds.functions 100 --coverage.thresholds.branches 100 --coverage.thresholds.statements 100`. + +##### coverage.thresholds[glob-pattern] + +- **Type:** `{ statements?: number functions?: number branches?: number lines?: number }` +- **Default:** `undefined` +- **Available for providers:** `'v8' | 'istanbul'` + +Sets thresholds for files matching the glob pattern. + + +```ts +{ + coverage: { + thresholds: { + // Thresholds for all files + functions: 95, + branches: 70, + + // Thresholds for matching glob pattern + 'src/utils/**.ts': { + statements: 95, + functions: 90, + branches: 85, + lines: 80, + }, + + // Files matching this pattern will only have lines thresholds set. + // Global thresholds are not inherited. + '**/math.ts': { + lines: 100, + } + } + } +} +``` #### coverage.ignoreClassMethods diff --git a/packages/coverage-istanbul/package.json b/packages/coverage-istanbul/package.json index 75c6f31219e3..e746a56b024a 100644 --- a/packages/coverage-istanbul/package.json +++ b/packages/coverage-istanbul/package.json @@ -50,6 +50,7 @@ "istanbul-lib-report": "^3.0.1", "istanbul-lib-source-maps": "^4.0.1", "istanbul-reports": "^3.1.6", + "magicast": "^0.3.2", "picocolors": "^1.0.0", "test-exclude": "^6.0.0" }, diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index b0443dd6a4ee..5d47794d4113 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -1,9 +1,10 @@ -import { existsSync, promises as fs } from 'node:fs' +import { existsSync, promises as fs, writeFileSync } from 'node:fs' import { resolve } from 'pathe' import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest' import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config' import { BaseCoverageProvider } from 'vitest/coverage' import c from 'picocolors' +import { parseModule } from 'magicast' import libReport from 'istanbul-lib-report' import reports from 'istanbul-reports' import type { CoverageMap, CoverageMapData } from 'istanbul-lib-coverage' @@ -65,10 +66,14 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co provider: 'istanbul', reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory), reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter), - lines: config['100'] ? 100 : config.lines, - functions: config['100'] ? 100 : config.functions, - branches: config['100'] ? 100 : config.branches, - statements: config['100'] ? 100 : config.statements, + + thresholds: config.thresholds && { + ...config.thresholds, + lines: config.thresholds['100'] ? 100 : config.thresholds.lines, + branches: config.thresholds['100'] ? 100 : config.thresholds.branches, + functions: config.thresholds['100'] ? 100 : config.thresholds.functions, + statements: config.thresholds['100'] ? 100 : config.thresholds.statements, + }, } this.instrumenter = createInstrumenter({ @@ -170,34 +175,36 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co }).execute(context) } - if (this.options.branches - || this.options.functions - || this.options.lines - || this.options.statements) { - this.checkThresholds({ + if (this.options.thresholds) { + const resolvedThresholds = this.resolveThresholds({ coverageMap, - thresholds: { - branches: this.options.branches, - functions: this.options.functions, - lines: this.options.lines, - statements: this.options.statements, - }, - perFile: this.options.perFile, + thresholds: this.options.thresholds, + createCoverageMap: () => libCoverage.createCoverageMap({}), }) - } - if (this.options.thresholdAutoUpdate && allTestsRun) { - this.updateThresholds({ - coverageMap, - thresholds: { - branches: this.options.branches, - functions: this.options.functions, - lines: this.options.lines, - statements: this.options.statements, - }, - perFile: this.options.perFile, - configurationFile: this.ctx.server.config.configFile, + this.checkThresholds({ + thresholds: resolvedThresholds, + perFile: this.options.thresholds.perFile, }) + + if (this.options.thresholds.autoUpdate && allTestsRun) { + if (!this.ctx.server.config.configFile) + throw new Error('Missing configurationFile. The "coverage.thresholds.autoUpdate" can only be enabled when configuration file is used.') + + const configFilePath = this.ctx.server.config.configFile + const configModule = parseModule(await fs.readFile(configFilePath, 'utf8')) + + this.updateThresholds({ + thresholds: resolvedThresholds, + perFile: this.options.thresholds.perFile, + configurationFile: { + write: () => writeFileSync(configFilePath, configModule.generate().code, 'utf-8'), + read: () => configModule.exports.default.$type === 'function-call' + ? configModule.exports.default.$args[0] + : configModule.exports.default, + }, + }) + } } } diff --git a/packages/coverage-v8/package.json b/packages/coverage-v8/package.json index d6f68b12f8b6..c0e93a6a0a8e 100644 --- a/packages/coverage-v8/package.json +++ b/packages/coverage-v8/package.json @@ -52,6 +52,7 @@ "istanbul-lib-source-maps": "^4.0.1", "istanbul-reports": "^3.1.6", "magic-string": "^0.30.5", + "magicast": "^0.3.2", "picocolors": "^1.0.0", "std-env": "^3.4.3", "test-exclude": "^6.0.0", diff --git a/packages/coverage-v8/src/provider.ts b/packages/coverage-v8/src/provider.ts index 0e8ee984356a..b88dc80af2e2 100644 --- a/packages/coverage-v8/src/provider.ts +++ b/packages/coverage-v8/src/provider.ts @@ -1,4 +1,4 @@ -import { existsSync, promises as fs } from 'node:fs' +import { existsSync, promises as fs, writeFileSync } from 'node:fs' import type { Profiler } from 'node:inspector' import { fileURLToPath, pathToFileURL } from 'node:url' import v8ToIstanbul from 'v8-to-istanbul' @@ -9,6 +9,7 @@ import type { CoverageMap, CoverageMapData } from 'istanbul-lib-coverage' import libCoverage from 'istanbul-lib-coverage' import libSourceMaps from 'istanbul-lib-source-maps' import MagicString from 'magic-string' +import { parseModule } from 'magicast' import remapping from '@ampproject/remapping' import { normalize, resolve } from 'pathe' import c from 'picocolors' @@ -72,10 +73,14 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage provider: 'v8', reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter), reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory), - lines: config['100'] ? 100 : config.lines, - functions: config['100'] ? 100 : config.functions, - branches: config['100'] ? 100 : config.branches, - statements: config['100'] ? 100 : config.statements, + + thresholds: config.thresholds && { + ...config.thresholds, + lines: config.thresholds['100'] ? 100 : config.thresholds.lines, + branches: config.thresholds['100'] ? 100 : config.thresholds.branches, + functions: config.thresholds['100'] ? 100 : config.thresholds.functions, + statements: config.thresholds['100'] ? 100 : config.thresholds.statements, + }, } this.testExclude = new _TestExclude({ @@ -156,34 +161,36 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage }).execute(context) } - if (this.options.branches - || this.options.functions - || this.options.lines - || this.options.statements) { - this.checkThresholds({ + if (this.options.thresholds) { + const resolvedThresholds = this.resolveThresholds({ coverageMap, - thresholds: { - branches: this.options.branches, - functions: this.options.functions, - lines: this.options.lines, - statements: this.options.statements, - }, - perFile: this.options.perFile, + thresholds: this.options.thresholds, + createCoverageMap: () => libCoverage.createCoverageMap({}), }) - } - if (this.options.thresholdAutoUpdate && allTestsRun) { - this.updateThresholds({ - coverageMap, - thresholds: { - branches: this.options.branches, - functions: this.options.functions, - lines: this.options.lines, - statements: this.options.statements, - }, - perFile: this.options.perFile, - configurationFile: this.ctx.server.config.configFile, + this.checkThresholds({ + thresholds: resolvedThresholds, + perFile: this.options.thresholds.perFile, }) + + if (this.options.thresholds.autoUpdate && allTestsRun) { + if (!this.ctx.server.config.configFile) + throw new Error('Missing configurationFile. The "coverage.thresholds.autoUpdate" can only be enabled when configuration file is used.') + + const configFilePath = this.ctx.server.config.configFile + const configModule = parseModule(await fs.readFile(configFilePath, 'utf8')) + + this.updateThresholds({ + thresholds: resolvedThresholds, + perFile: this.options.thresholds.perFile, + configurationFile: { + write: () => writeFileSync(configFilePath, configModule.generate().code, 'utf-8'), + read: () => configModule.exports.default.$type === 'function-call' + ? configModule.exports.default.$args[0] + : configModule.exports.default, + }, + }) + } } } diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index a6dad4e46436..c6f453e90a01 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -25,7 +25,7 @@ cli .option('--hideSkippedTests', 'Hide logs for skipped tests') .option('--reporter ', 'Specify reporters') .option('--outputFile ', 'Write test results to a file when supporter reporter is also specified, use cac\'s dot notation for individual outputs of multiple reporters') - .option('--coverage', 'Enable coverage report', { default: { 100: false } }) + .option('--coverage', 'Enable coverage report') .option('--run', 'Disable watch mode') .option('--mode ', 'Override Vite mode (default: test)') .option('--globals', 'Inject apis globally') diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index 6f0be7da13f1..bcd96182ef9b 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -157,40 +157,27 @@ export interface BaseCoverageOptions { skipFull?: boolean /** - * Check thresholds per file. - * See `lines`, `functions`, `branches` and `statements` for the actual thresholds. + * Configurations for thresholds * - * @default false - */ - perFile?: boolean - - /** - * Threshold for lines - * - * @default undefined - */ - lines?: number - - /** - * Threshold for functions - * - * @default undefined - */ - functions?: number - - /** - * Threshold for branches + * @example * - * @default undefined - */ - branches?: number - - /** - * Threshold for statements + * ```ts + * { + * // Thresholds for all files + * functions: 95, + * branches: 70, + * perFile: true, + * autoUpdate: true, * - * @default undefined + * // Thresholds for utilities + * 'src/utils/**.ts': { + * lines: 100, + * statements: 95, + * } + * } + * ``` */ - statements?: number + thresholds?: Thresholds | ({ [glob: string]: Pick } & Thresholds) /** * Watermarks for statements, lines, branches and functions. @@ -204,13 +191,6 @@ export interface BaseCoverageOptions { lines?: [number, number] } - /** - * Update threshold values automatically when current coverage is higher than earlier thresholds - * - * @default false - */ - thresholdAutoUpdate?: boolean - /** * Generate coverage report even when tests fail. * @@ -224,13 +204,6 @@ export interface BaseCoverageOptions { * @default false */ allowExternal?: boolean - - /** - * Shortcut for `{ lines: 100, functions: 100, branches: 100, statements: 100 }` - * - * @default false - */ - 100?: boolean } export interface CoverageIstanbulOptions extends BaseCoverageOptions { @@ -248,3 +221,30 @@ export interface CustomProviderOptions extends Pick> +} + const THRESHOLD_KEYS: Readonly = ['lines', 'functions', 'statements', 'branches'] +const GLOBAL_THRESHOLDS_KEY = 'global' export class BaseCoverageProvider { /** * Check if current coverage is above configured thresholds and bump the thresholds if needed */ - updateThresholds({ configurationFile, coverageMap, thresholds, perFile }: { - coverageMap: CoverageMap - thresholds: Record + updateThresholds({ thresholds: allThresholds, perFile, configurationFile }: { + thresholds: ResolvedThreshold[] perFile?: boolean - configurationFile?: string + configurationFile: { read(): unknown; write(): void } }) { - // Thresholds cannot be updated if there is no configuration file and - // feature was enabled by CLI, e.g. --coverage.thresholdAutoUpdate - if (!configurationFile) - throw new Error('Missing configurationFile. The "coverage.thresholdAutoUpdate" can only be enabled when configuration file is used.') + let updatedThresholds = false - const summaries = perFile - ? coverageMap.files() - .map((file: string) => coverageMap.fileCoverageFor(file).toSummary()) - : [coverageMap.getCoverageSummary()] + const config = configurationFile.read() + assertConfigurationModule(config) - const thresholdsToUpdate: [Threshold, number][] = [] + for (const { coverageMap, thresholds, name } of allThresholds) { + const summaries = perFile + ? coverageMap.files() + .map((file: string) => coverageMap.fileCoverageFor(file).toSummary()) + : [coverageMap.getCoverageSummary()] - for (const key of THRESHOLD_KEYS) { - const threshold = thresholds[key] ?? 100 - const actual = Math.min(...summaries.map(summary => summary[key].pct)) + const thresholdsToUpdate: [Threshold, number][] = [] - if (actual > threshold) - thresholdsToUpdate.push([key, actual]) - } + for (const key of THRESHOLD_KEYS) { + const threshold = thresholds[key] ?? 100 + const actual = Math.min(...summaries.map(summary => summary[key].pct)) - if (thresholdsToUpdate.length === 0) - return + if (actual > threshold) + thresholdsToUpdate.push([key, actual]) + } - const originalConfig = readFileSync(configurationFile, 'utf8') - let updatedConfig = originalConfig + if (thresholdsToUpdate.length === 0) + continue - for (const [threshold, newValue] of thresholdsToUpdate) { - // Find the exact match from the configuration file and replace the value - const previousThreshold = (thresholds[threshold] ?? 100).toString() - const pattern = new RegExp(`(${threshold}\\s*:\\s*)${previousThreshold.replace('.', '\\.')}`) - const matches = originalConfig.match(pattern) + updatedThresholds = true - if (matches) - updatedConfig = updatedConfig.replace(matches[0], matches[1] + newValue) - else - console.error(`Unable to update coverage threshold ${threshold}. No threshold found using pattern ${pattern}`) + for (const [threshold, newValue] of thresholdsToUpdate) { + if (name === GLOBAL_THRESHOLDS_KEY) { + config.test.coverage.thresholds[threshold] = newValue + } + else { + const glob = config.test.coverage.thresholds[name as Threshold] as ResolvedThreshold['thresholds'] + glob[threshold] = newValue + } + } } - if (updatedConfig !== originalConfig) { + if (updatedThresholds) { // eslint-disable-next-line no-console console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.') - writeFileSync(configurationFile, updatedConfig, 'utf-8') + configurationFile.write() } } /** - * Checked collected coverage against configured thresholds. Sets exit code to 1 when thresholds not reached. + * Check collected coverage against configured thresholds. Sets exit code to 1 when thresholds not reached. */ - checkThresholds({ coverageMap, thresholds, perFile }: { - coverageMap: CoverageMap - thresholds: Record - perFile?: boolean - }) { - // Construct list of coverage summaries where thresholds are compared against - const summaries = perFile - ? coverageMap.files() - .map((file: string) => ({ - file, - summary: coverageMap.fileCoverageFor(file).toSummary(), - })) - : [{ - file: null, - summary: coverageMap.getCoverageSummary(), - }] - - // Check thresholds of each summary - for (const { summary, file } of summaries) { - for (const thresholdKey of ['lines', 'functions', 'statements', 'branches'] as const) { - const threshold = thresholds[thresholdKey] - - if (threshold !== undefined) { - const coverage = summary.data[thresholdKey].pct - - if (coverage < threshold) { - process.exitCode = 1 - - /* - * Generate error message based on perFile flag: - * - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts - * - ERROR: Coverage for statements (50%) does not meet global threshold (85%) - */ - let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet` - - if (!perFile) - errorMessage += ' global' - - errorMessage += ` threshold (${threshold}%)` - - if (perFile && file) - errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` - - console.error(errorMessage) + checkThresholds({ thresholds: allThresholds, perFile }: { thresholds: ResolvedThreshold[]; perFile?: boolean }) { + for (const { coverageMap, thresholds, name } of allThresholds) { + if (thresholds.branches === undefined + && thresholds.functions === undefined + && thresholds.lines === undefined + && thresholds.statements === undefined) + continue + + // Construct list of coverage summaries where thresholds are compared against + const summaries = perFile + ? coverageMap.files() + .map((file: string) => ({ + file, + summary: coverageMap.fileCoverageFor(file).toSummary(), + })) + : [{ + file: null, + summary: coverageMap.getCoverageSummary(), + }] + + // Check thresholds of each summary + for (const { summary, file } of summaries) { + for (const thresholdKey of ['lines', 'functions', 'statements', 'branches'] as const) { + const threshold = thresholds[thresholdKey] + + if (threshold !== undefined) { + const coverage = summary.data[thresholdKey].pct + + if (coverage < threshold) { + process.exitCode = 1 + + /* + * Generate error message based on perFile flag: + * - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts + * - ERROR: Coverage for statements (50%) does not meet global threshold (85%) + */ + let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`} threshold (${threshold}%)` + + if (perFile && file) + errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}` + + console.error(errorMessage) + } } } } } } + /** + * Constructs collected coverage and users' threshold options into separate sets + * where each threshold set holds their own coverage maps. Threshold set is either + * for specific files defined by glob pattern or global for all other files. + */ + resolveThresholds({ coverageMap, thresholds, createCoverageMap }: { + coverageMap: CoverageMap + thresholds: NonNullable + createCoverageMap: () => CoverageMap + }): ResolvedThreshold[] { + const resolvedThresholds: ResolvedThreshold[] = [] + const files = coverageMap.files() + const filesMatchedByGlobs: string[] = [] + const globalCoverageMap = createCoverageMap() + + for (const key of Object.keys(thresholds) as (`${keyof typeof thresholds}`[])) { + if (key === 'perFile' || key === 'autoUpdate' || key === '100' || THRESHOLD_KEYS.includes(key)) + continue + + const glob = key + const globThresholds = resolveGlobThresholds(thresholds[glob]) + const globCoverageMap = createCoverageMap() + + const matchingFiles = files.filter(file => mm.isMatch(file, glob)) + filesMatchedByGlobs.push(...matchingFiles) + + for (const file of matchingFiles) { + const fileCoverage = coverageMap.fileCoverageFor(file) + globCoverageMap.addFileCoverage(fileCoverage) + } + + resolvedThresholds.push({ + name: glob, + coverageMap: globCoverageMap, + thresholds: globThresholds, + }) + } + + // Global threshold is for all files that were not included by glob patterns + for (const file of files.filter(file => !filesMatchedByGlobs.includes(file))) { + const fileCoverage = coverageMap.fileCoverageFor(file) + globalCoverageMap.addFileCoverage(fileCoverage) + } + + resolvedThresholds.unshift({ + name: GLOBAL_THRESHOLDS_KEY, + coverageMap: globalCoverageMap, + thresholds: { + branches: thresholds.branches, + functions: thresholds.functions, + lines: thresholds.lines, + statements: thresholds.statements, + }, + }) + + return resolvedThresholds + } + /** * Resolve reporters from various configuration options */ @@ -139,3 +201,30 @@ export class BaseCoverageProvider { return resolvedReporters } } + +/** + * Narrow down `unknown` glob thresholds to resolved ones + */ +function resolveGlobThresholds(thresholds: unknown): ResolvedThreshold['thresholds'] { + if (!thresholds || typeof thresholds !== 'object') + return { } + + return { + lines: 'lines' in thresholds && typeof thresholds.lines === 'number' ? thresholds.lines : undefined, + branches: 'branches' in thresholds && typeof thresholds.branches === 'number' ? thresholds.branches : undefined, + functions: 'functions' in thresholds && typeof thresholds.functions === 'number' ? thresholds.functions : undefined, + statements: 'statements' in thresholds && typeof thresholds.statements === 'number' ? thresholds.statements : undefined, + } +} + +function assertConfigurationModule(config: unknown): asserts config is { test: { coverage: { thresholds: NonNullable } } } { + try { + // @ts-expect-error -- Intentional unsafe null pointer check as wrapped in try-catch + if (typeof config.test.coverage.thresholds !== 'object') + throw new Error('Expected config.test.coverage.thresholds to be an object') + } + catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Unable to parse thresholds from configuration file: ${message}`) + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d7c3c631f0d..113fd8da91fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -929,6 +929,9 @@ importers: istanbul-reports: specifier: ^3.1.6 version: 3.1.6 + magicast: + specifier: ^0.3.2 + version: 0.3.2 picocolors: specifier: ^1.0.0 version: 1.0.0 @@ -981,6 +984,9 @@ importers: magic-string: specifier: ^0.30.5 version: 0.30.5 + magicast: + specifier: ^0.3.2 + version: 0.3.2 picocolors: specifier: ^1.0.0 version: 1.0.0 @@ -1569,6 +1575,9 @@ importers: istanbul-lib-coverage: specifier: ^3.2.0 version: 3.2.0 + magicast: + specifier: ^0.3.2 + version: 0.3.2 vite: specifier: ^5.0.0-beta.19 version: 5.0.0-beta.19(@types/node@18.18.9)(less@4.1.3) @@ -19897,6 +19906,13 @@ packages: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + /magicast@0.3.2: + resolution: {integrity: sha512-Fjwkl6a0syt9TFN0JSYpOybxiMCkYNEeOTnOTNRbjphirLakznZXAqrXgj/7GG3D1dvETONNwrBfinvAbpunDg==} + dependencies: + '@babel/parser': 7.23.3 + '@babel/types': 7.23.3 + source-map-js: 1.0.2 + /make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} diff --git a/test/config/test/failures.test.ts b/test/config/test/failures.test.ts index 64133128548b..31cb4194ba8b 100644 --- a/test/config/test/failures.test.ts +++ b/test/config/test/failures.test.ts @@ -81,6 +81,29 @@ test('boolean coverage flag without dot notation, with more dot notation options expect(stderr).toMatch('Please specify the "--coverage" argument with dot notation as well: "--coverage.enabled"') }) +test('coverage.autoUpdate cannot update thresholds when configuration file doesnt define them', async () => { + const { stderr } = await runVitest({ + coverage: { + enabled: true, + thresholds: { + autoUpdate: true, + lines: 0, + }, + }, + }) + + expect(stderr).toMatch('Error: Unable to parse thresholds from configuration file: Cannot read properties of undefined') +}) + +test('boolean flag 100 should not crash CLI', async () => { + const { stderr } = await runVitestCli('--coverage.enabled', '--coverage.thresholds.100') + + expect(stderr).toMatch('ERROR: Coverage for lines (0%) does not meet global threshold (100%)') + expect(stderr).toMatch('ERROR: Coverage for functions (0%) does not meet global threshold (100%)') + expect(stderr).toMatch('ERROR: Coverage for statements (0%) does not meet global threshold (100%)') + expect(stderr).toMatch('ERROR: Coverage for branches (0%) does not meet global threshold (100%)') +}) + test('boolean browser flag without dot notation, with more dot notation options', async () => { const { stderr } = await runVitestCli('run', '--browser', '--browser.name', 'chrome') diff --git a/test/config/vitest.config.ts b/test/config/vitest.config.ts index 0fb0f723dc01..7161494d71ba 100644 --- a/test/config/vitest.config.ts +++ b/test/config/vitest.config.ts @@ -1,6 +1,7 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ + // Note: One test expects coverage.thresholds not to be defined in here test: { include: ['test/**.test.ts'], reporters: ['verbose'], diff --git a/test/coverage-test/coverage-report-tests/__snapshots__/v8.report.test.ts.snap b/test/coverage-test/coverage-report-tests/__snapshots__/v8.report.test.ts.snap index 76791058c68c..941d848ed151 100644 --- a/test/coverage-test/coverage-report-tests/__snapshots__/v8.report.test.ts.snap +++ b/test/coverage-test/coverage-report-tests/__snapshots__/v8.report.test.ts.snap @@ -3315,7 +3315,7 @@ exports[`v8 json report 1`] = ` 0, ], "10": [ - 1, + 2, ], "2": [ 0, @@ -3324,7 +3324,7 @@ exports[`v8 json report 1`] = ` 1, ], "4": [ - 1, + 3, ], "5": [ 0, @@ -3336,10 +3336,10 @@ exports[`v8 json report 1`] = ` 1, ], "8": [ - 0, + 1, ], "9": [ - 0, + 1, ], }, "branchMap": { diff --git a/test/coverage-test/coverage-report-tests/generic.report.test.ts b/test/coverage-test/coverage-report-tests/generic.report.test.ts index 2ad012666812..5ae46824fc97 100644 --- a/test/coverage-test/coverage-report-tests/generic.report.test.ts +++ b/test/coverage-test/coverage-report-tests/generic.report.test.ts @@ -4,6 +4,7 @@ import fs from 'node:fs' import { resolve } from 'pathe' +import { parseModule } from 'magicast' import { expect, test } from 'vitest' import libCoverage from 'istanbul-lib-coverage' @@ -72,23 +73,46 @@ test('files should not contain a setup file', () => { expect(srcFiles).not.toContain('another-setup.ts.html') }) -test('thresholdAutoUpdate updates thresholds', async () => { +test('thresholds.autoUpdate updates thresholds', async () => { const configFilename = resolve('./vitest.config.ts') - const configContents = fs.readFileSync(configFilename, 'utf-8') + const mod = parseModule(fs.readFileSync(configFilename, 'utf-8')) + const thresholds = mod.exports.default.$args[0].test.coverage.thresholds - for (const threshold of ['functions', 'branches', 'lines', 'statements']) { - const match = configContents.match(new RegExp(`${threshold}: (?[\\d|\\.]+)`)) - const coverage = match?.groups?.coverage || '0' + // Configuration has fixed value of 1.01 and 0 set for each threshold + expect(Number.parseInt(thresholds.functions)).toBeGreaterThan(1.01) + expect(Number.parseInt(thresholds.branches)).toBeGreaterThan(1.01) + expect(Number.parseInt(thresholds.lines)).toBeGreaterThan(1.01) + expect(Number.parseInt(thresholds.statements)).toBeGreaterThan(1.01) - // Configuration has fixed value of 1.01 and 0 set for each threshold - expect(Number.parseInt(coverage)).toBeGreaterThan(1.01) + // Check file coverage for glob + const coverageJson = await readCoverageJson() + const coverageMap = libCoverage.createCoverageMap(coverageJson as any) + + const fileCoverage = coverageMap.fileCoverageFor('/src/function-count.ts') + const summary = fileCoverage.toSummary() + expect(summary.branches.pct).toBe(100) + expect(summary.functions.pct).toBe(60) + + if (process.env.COVERAGE_PROVIDER === 'v8') { + expect(summary.statements.pct).toBe(86.11) + expect(summary.lines.pct).toBe(86.11) + } + else { + expect(summary.statements.pct).toBe(71.42) + expect(summary.lines.pct).toBe(71.42) } // Update thresholds back to fixed values - const updatedConfig = configContents - .replace(/(branches|statements): ([\d|\.])+/g, '$1: 1.01') - .replace(/(functions|lines): ([\d|\.])+/g, '$1: 0') - fs.writeFileSync(configFilename, updatedConfig) + thresholds.functions = 0 + thresholds.lines = 0 + thresholds.branches = 1.01 + thresholds.statements = 1.01 + thresholds['**/function-count.ts'].statements = 50 + thresholds['**/function-count.ts'].branches = 99 + thresholds['**/function-count.ts'].functions = 59 + thresholds['**/function-count.ts'].lines = 50 + + fs.writeFileSync(configFilename, mod.generate().code, 'utf-8') }) test('function count is correct', async () => { diff --git a/test/coverage-test/option-tests/threshold-100.test.ts b/test/coverage-test/option-tests/threshold-100.test.ts new file mode 100644 index 000000000000..56caafad2e47 --- /dev/null +++ b/test/coverage-test/option-tests/threshold-100.test.ts @@ -0,0 +1,18 @@ +// eslint-disable-next-line ts/ban-ts-comment -- Type tests keep picking this file up and fails +// @ts-nocheck + +import { assert, expect, test } from 'vitest' +import { getWorkerState } from 'vitest/src/utils.js' + +test('thresholds.100 sets global thresholds to 100', () => { + const state = getWorkerState() + + assert(state.config.coverage.provider === 'v8' || state.config.coverage.provider === 'istanbul') + assert(state.config.coverage.thresholds !== undefined) + + expect(state.config.coverage.thresholds[100]).toBe(true) + expect(state.config.coverage.thresholds.lines).toBe(100) + expect(state.config.coverage.thresholds.branches).toBe(100) + expect(state.config.coverage.thresholds.functions).toBe(100) + expect(state.config.coverage.thresholds.statements).toBe(100) +}) diff --git a/test/coverage-test/package.json b/test/coverage-test/package.json index d9ecd9b6e037..fbdddf72c679 100644 --- a/test/coverage-test/package.json +++ b/test/coverage-test/package.json @@ -21,6 +21,7 @@ "@vue/test-utils": "latest", "happy-dom": "latest", "istanbul-lib-coverage": "^3.2.0", + "magicast": "^0.3.2", "vite": "latest", "vitest": "workspace:*", "vue": "latest", diff --git a/test/coverage-test/test/configuration-options.test-d.ts b/test/coverage-test/test/configuration-options.test-d.ts index 68a88248a415..8af873a44229 100644 --- a/test/coverage-test/test/configuration-options.test-d.ts +++ b/test/coverage-test/test/configuration-options.test-d.ts @@ -30,7 +30,20 @@ test('provider options, generic', () => { functions: [80, 95], lines: [80, 95], }, - 100: true, + thresholds: { + '100': true, + 'lines': 1, + 'autoUpdate': true, + 'perFile': true, + 'statements': 100, + + '**/some-file.ts': { + lines: 12, + branches: 12, + functions: 12, + statements: 12, + }, + }, }) assertType({ @@ -40,7 +53,16 @@ test('provider options, generic', () => { watermarks: { statements: [80, 95], }, - 100: true, + thresholds: { + '100': true, + + '**/some-file.ts': { + lines: 12, + branches: 12, + functions: 12, + statements: 12, + }, + }, }) }) diff --git a/test/coverage-test/testing-options.mjs b/test/coverage-test/testing-options.mjs index f40430fcfa78..cf1150382979 100644 --- a/test/coverage-test/testing-options.mjs +++ b/test/coverage-test/testing-options.mjs @@ -31,6 +31,18 @@ const testCases = [ include: ['coverage-report-tests/allow-external.test.ts'], }, }, + { + testConfig: { + name: 'thresholds.100', + include: ['option-tests/threshold-100.test.ts'], + coverage: { + thresholds: { + 100: true, + }, + }, + }, + assertionConfig: null, + }, ] for (const provider of ['v8', 'istanbul']) { @@ -52,6 +64,9 @@ for (const provider of ['v8', 'istanbul']) { checkExit() + if (!assertionConfig) + continue + // Check generated coverage report await startVitest('test', ['coverage-report-tests'], { config: false, diff --git a/test/coverage-test/vitest.config.ts b/test/coverage-test/vitest.config.ts index 7f5fc5c7a65c..a95ca0ed051d 100644 --- a/test/coverage-test/vitest.config.ts +++ b/test/coverage-test/vitest.config.ts @@ -82,11 +82,21 @@ export default defineConfig({ ], // These will be updated by tests and reseted back by generic.report.test.ts - thresholdAutoUpdate: true, - functions: 0, - branches: 1.01, - lines: 0, - statements: 1.01, + thresholds: { + 'autoUpdate': true, + 'functions': 0, + 'branches': 1.01, + 'lines': 0, + 'statements': 1.01, + + // These need to pass both V8 and istanbul + '**/function-count.ts': { + statements: 50, + branches: 99, + functions: 59, + lines: 50, + }, + }, }, setupFiles: [ resolve(__dirname, './setup.ts'),