feat(coverage): automatic threshold updating

packages/coverage-c8/package.json

@@ -45,6 +45,7 @@
     "vitest": ">=0.29.0 <1"
   },
   "dependencies": {
+    "@vitest/utils": "workspace:*",
     "c8": "^7.12.0",
     "picocolors": "^1.0.0",
     "std-env": "^3.3.1"

packages/coverage-c8/src/provider.ts

@@ -6,6 +6,7 @@ import c from 'picocolors'
 import { provider } from 'std-env'
 import type { RawSourceMap } from 'vite-node'
 import { coverageConfigDefaults } from 'vitest/config'
+import { updateThresholds } from '@vitest/utils/coverage'
 // eslint-disable-next-line no-restricted-imports
 import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest'
 import type { Vitest } from 'vitest/node'
@@ -147,6 +148,19 @@ export class C8CoverageProvider implements CoverageProvider {
 
     await report.run()
     await checkCoverages(options, report)
+
+    if (this.options.thresholdAutoUpdate && allTestsRun) {
+      updateThresholds({
+        coverageMap: await report.getCoverageMapFromAllCoverageFiles(),
+        thresholds: {
+          branches: this.options.branches,
+          functions: this.options.functions,
+          lines: this.options.lines,
+          statements: this.options.statements,
+        },
+        configurationFile: this.ctx.server.config.configFile,
+      })
+    }
   }
 }
 

packages/coverage-istanbul/package.json

@@ -45,6 +45,7 @@
     "vitest": ">=0.28.0 <1"
   },
   "dependencies": {
+    "@vitest/utils": "workspace:*",
     "istanbul-lib-coverage": "^3.2.0",
     "istanbul-lib-instrument": "^5.2.1",
     "istanbul-lib-report": "^3.0.0",

packages/coverage-istanbul/src/provider.ts

@@ -4,6 +4,7 @@ import { relative, resolve } from 'pathe'
 import type { TransformPluginContext } from 'rollup'
 import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
 import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config'
+import { updateThresholds } from '@vitest/utils/coverage'
 import libReport from 'istanbul-lib-report'
 import reports from 'istanbul-reports'
 import type { CoverageMap } from 'istanbul-lib-coverage'
@@ -140,6 +141,19 @@ export class IstanbulCoverageProvider implements CoverageProvider {
         statements: this.options.statements,
       })
     }
+
+    if (this.options.thresholdAutoUpdate && allTestsRun) {
+      updateThresholds({
+        coverageMap,
+        thresholds: {
+          branches: this.options.branches,
+          functions: this.options.functions,
+          lines: this.options.lines,
+          statements: this.options.statements,
+        },
+        configurationFile: this.ctx.server.config.configFile,
+      })
+    }
   }
 
   checkThresholds(coverageMap: CoverageMap, thresholds: Record<Threshold, number | undefined>) {

packages/utils/coverage.d.ts

@@ -0,0 +1 @@
+export * from './dist/coverage.js'

packages/utils/package.json

@@ -23,6 +23,10 @@
       "types": "./dist/helpers.d.ts",
       "import": "./dist/helpers.js"
     },
+    "./coverage": {
+      "types": "./dist/coverage.d.ts",
+      "import": "./dist/coverage.js"
+    },
     "./*": "./*"
   },
   "main": "./dist/index.js",
@@ -44,6 +48,7 @@
     "pretty-format": "^27.5.1"
   },
   "devDependencies": {
-    "@types/diff": "^5.0.2"
+    "@types/diff": "^5.0.2",
+    "@types/istanbul-lib-coverage": "^2.0.4"
   }
 }

packages/utils/rollup.config.js

@@ -11,6 +11,7 @@ const entries = {
   helpers: 'src/helpers.ts',
   diff: 'src/diff.ts',
   types: 'src/types.ts',
+  coverage: 'src/coverage.ts',
 }
 
 const external = [

packages/utils/src/coverage.ts

@@ -0,0 +1,50 @@
+import { readFileSync, writeFileSync } from 'node:fs'
+import type { CoverageMap } from 'istanbul-lib-coverage'
+
+type Threshold = 'lines' | 'functions' | 'statements' | 'branches'
+
+const THRESHOLD_KEYS: Readonly<Threshold[]> = ['lines', 'functions', 'statements', 'branches']
+
+/**
+ * Check if current coverage is above configured thresholds and bump the thresholds if needed
+ */
+export function updateThresholds({ configurationFile, coverageMap, thresholds }: {
+  coverageMap: CoverageMap
+  thresholds: Record<Threshold, number | undefined>
+  configurationFile?: string
+}) {
+  if (!configurationFile)
+    throw new Error('Missing configurationFile')
+
+  const summary = coverageMap.getCoverageSummary()
+  const thresholdsToUpdate: Threshold[] = []
+
+  for (const key of THRESHOLD_KEYS) {
+    const threshold = thresholds[key] || 100
+    const actual = summary[key].pct
+
+    if (actual > threshold)
+      thresholdsToUpdate.push(key)
+  }
+
+  if (thresholdsToUpdate.length === 0)
+    return
+
+  const originalConfig = readFileSync(configurationFile, 'utf8')
+  let updatedConfig = originalConfig
+
+  for (const threshold of thresholdsToUpdate) {
+    const matches = originalConfig.match(new RegExp(`(${threshold}\\s*:\\s*)[\\d|\\.]+`))
+
+    if (matches)
+      updatedConfig = updatedConfig.replace(matches[0], matches[1] + summary[threshold].pct)
+    else
+      console.error(`Unable to update coverage threshold ${threshold}`)
+  }
+
+  if (updatedConfig !== originalConfig) {
+    // eslint-disable-next-line no-console
+    console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.')
+    writeFileSync(configurationFile, updatedConfig, 'utf-8')
+  }
+}

packages/vitest/src/types/coverage.ts

@@ -192,6 +192,13 @@ export interface BaseCoverageOptions {
    * @default undefined
    */
   statements?: number
+
+  /**
+   * Update threshold values automatically when current coverage is higher than earlier thresholds
+   *
+   * @default false
+   */
+  thresholdAutoUpdate?: boolean
 }
 
 export interface CoverageIstanbulOptions extends BaseCoverageOptions {

test/coverage-test/coverage-report-tests/generic.report.test.ts

@@ -63,3 +63,20 @@ test('files should not contain a setup file', () => {
 
   expect(srcFiles).not.toContain('another-setup.ts.html')
 })
+
+test('thresholdAutoUpdate updates thresholds', async () => {
+  const configFilename = resolve('./vitest.config.ts')
+  const configContents = fs.readFileSync(configFilename, 'utf-8')
+
+  for (const threshold of ['functions', 'branches', 'lines', 'statements']) {
+    const match = configContents.match(new RegExp(`${threshold}: (?<coverage>[\\d|\\.]+)`))
+    const coverage = match?.groups?.coverage || '0'
+
+    // Configuration has fixed value of 1.01 set for each threshold
+    expect(parseInt(coverage)).toBeGreaterThan(1.01)
+  }
+
+  // Update thresholds back to fixed values
+  const updatedConfig = configContents.replace(/(branches|functions|lines|statements): ([\d|\.])+/g, '$1: 1.01')
+  fs.writeFileSync(configFilename, updatedConfig)
+})

test/coverage-test/vitest.config.ts

@@ -20,6 +20,13 @@ export default defineConfig({
       clean: true,
       all: true,
       reporter: ['html', 'text', 'lcov', 'json'],
+
+      // These will be updated by tests and reseted back by generic.report.test.ts
+      thresholdAutoUpdate: true,
+      functions: 1.01,
+      branches: 1.01,
+      lines: 1.01,
+      statements: 1.01,
     },
     setupFiles: [
       resolve(__dirname, './setup.ts'),