feat(coverage): support --changed option (#5314)

Author: AriPerkkio

docs/guide/cli.md

@@ -137,6 +137,8 @@ vitest --api=false
 
   To run tests against changes made in the last commit, you can use `--changed HEAD~1`. You can also pass commit hash (e.g. `--changed 09a9920`) or branch name (e.g. `--changed origin/develop`).
 
+  When used with code coverage the report will contain only the files that were related to the changes.
+
   If paired with the [`forceRerunTriggers`](/config/#forcereruntriggers) config option it will run the whole test suite if at least one of the files listed in the `forceRerunTriggers` list changes. By default, changes to the Vitest config file and `package.json` will always rerun the whole suite.
 
 ### shard

packages/coverage-istanbul/src/provider.ts

@@ -253,27 +253,25 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
   }
 
   async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
-    // Load, instrument and collect empty coverages from all files which
-    // are not already in the coverage map
-    const includedFiles = await this.testExclude.glob(this.ctx.config.root)
+    const allFiles = await this.testExclude.glob(this.ctx.config.root)
+    let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
+
+    if (this.ctx.config.changed)
+      includedFiles = (this.ctx.config.related || []).filter(file => includedFiles.includes(file))
+
     const uncoveredFiles = includedFiles
-      .map(file => resolve(this.ctx.config.root, file))
       .filter(file => !coveredFiles.includes(file))
 
+    const cacheKey = new Date().getTime()
     const coverageMap = libCoverage.createCoverageMap({})
 
     // Note that these cannot be run parallel as synchronous instrumenter.lastFileCoverage
     // returns the coverage of the last transformed file
     for (const [index, filename] of uncoveredFiles.entries()) {
       debug('Uncovered file %s %d/%d', filename, index, uncoveredFiles.length)
 
-      // Make sure file is not served from cache
-      // so that instrumenter loads up requested file coverage
-      if (this.ctx.vitenode.fetchCache.has(filename))
-        this.ctx.vitenode.fetchCache.delete(filename)
-
-      await this.ctx.vitenode.transformRequest(filename)
-
+      // Make sure file is not served from cache so that instrumenter loads up requested file coverage
+      await this.ctx.vitenode.transformRequest(`${filename}?v=${cacheKey}`)
       const lastCoverage = this.instrumenter.lastFileCoverage()
       coverageMap.addFileCoverage(lastCoverage)
     }

packages/coverage-v8/src/provider.ts

@@ -246,9 +246,14 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
   private async getUntestedFiles(testedFiles: string[]): Promise<RawCoverage> {
     const transformResults = normalizeTransformResults(this.ctx.vitenode.fetchCache)
 
-    const includedFiles = await this.testExclude.glob(this.ctx.config.root)
+    const allFiles = await this.testExclude.glob(this.ctx.config.root)
+    let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
+
+    if (this.ctx.config.changed)
+      includedFiles = (this.ctx.config.related || []).filter(file => includedFiles.includes(file))
+
     const uncoveredFiles = includedFiles
-      .map(file => pathToFileURL(resolve(this.ctx.config.root, file)))
+      .map(file => pathToFileURL(file))
       .filter(file => !testedFiles.includes(file.pathname))
 
     let merged: RawCoverage = { result: [] }

packages/vitest/src/node/core.ts

@@ -382,10 +382,6 @@ export class Vitest {
       }
     }
 
-    // all subsequent runs will treat this as a fresh run
-    this.config.changed = false
-    this.config.related = undefined
-
     if (files.length) {
       // populate once, update cache on watch
       await this.cache.stats.populateStats(this.config.root, files)
@@ -532,6 +528,10 @@ export class Vitest {
 
         this.runningPromise = undefined
         this.isFirstRun = false
+
+        // all subsequent runs will treat this as a fresh run
+        this.config.changed = false
+        this.config.related = undefined
       })
 
     return await this.runningPromise

test/coverage-test/coverage-report-tests/__snapshots__/istanbul.report.test.ts.snap

@@ -1134,6 +1134,89 @@ exports[`istanbul json report 1`] = `
       },
     },
   },
+  "<process-cwd>/src/file-to-change.ts": {
+    "b": {},
+    "branchMap": {},
+    "f": {
+      "0": 0,
+      "1": 0,
+    },
+    "fnMap": {
+      "0": {
+        "decl": {
+          "end": {
+            "column": 22,
+            "line": 1,
+          },
+          "start": {
+            "column": 16,
+            "line": 1,
+          },
+        },
+        "loc": {
+          "end": {
+            "column": null,
+            "line": 3,
+          },
+          "start": {
+            "column": 22,
+            "line": 1,
+          },
+        },
+        "name": "run",
+      },
+      "1": {
+        "decl": {
+          "end": {
+            "column": 36,
+            "line": 5,
+          },
+          "start": {
+            "column": 16,
+            "line": 5,
+          },
+        },
+        "loc": {
+          "end": {
+            "column": null,
+            "line": 7,
+          },
+          "start": {
+            "column": 36,
+            "line": 5,
+          },
+        },
+        "name": "uncoveredFunction",
+      },
+    },
+    "path": "<process-cwd>/src/file-to-change.ts",
+    "s": {
+      "0": 0,
+      "1": 0,
+    },
+    "statementMap": {
+      "0": {
+        "end": {
+          "column": null,
+          "line": 2,
+        },
+        "start": {
+          "column": 2,
+          "line": 2,
+        },
+      },
+      "1": {
+        "end": {
+          "column": null,
+          "line": 6,
+        },
+        "start": {
+          "column": 2,
+          "line": 6,
+        },
+      },
+    },
+  },
   "<process-cwd>/src/function-count.ts": {
     "b": {},
     "branchMap": {},

test/coverage-test/coverage-report-tests/__snapshots__/v8.report.test.ts.snap

@@ -2777,6 +2777,153 @@ exports[`v8 json report 1`] = `
       },
     },
   },
+  "<process-cwd>/src/file-to-change.ts": {
+    "all": true,
+    "b": {
+      "0": [
+        0,
+      ],
+    },
+    "branchMap": {
+      "0": {
+        "line": 1,
+        "loc": {
+          "end": {
+            "column": 1,
+            "line": 7,
+          },
+          "start": {
+            "column": 0,
+            "line": 1,
+          },
+        },
+        "locations": [
+          {
+            "end": {
+              "column": 1,
+              "line": 7,
+            },
+            "start": {
+              "column": 0,
+              "line": 1,
+            },
+          },
+        ],
+        "type": "branch",
+      },
+    },
+    "f": {
+      "0": 0,
+    },
+    "fnMap": {
+      "0": {
+        "decl": {
+          "end": {
+            "column": 1,
+            "line": 7,
+          },
+          "start": {
+            "column": 0,
+            "line": 1,
+          },
+        },
+        "line": 1,
+        "loc": {
+          "end": {
+            "column": 1,
+            "line": 7,
+          },
+          "start": {
+            "column": 0,
+            "line": 1,
+          },
+        },
+        "name": "(empty-report)",
+      },
+    },
+    "path": "<process-cwd>/src/file-to-change.ts",
+    "s": {
+      "0": 0,
+      "1": 0,
+      "2": 0,
+      "3": 0,
+      "4": 0,
+      "5": 0,
+      "6": 0,
+    },
+    "statementMap": {
+      "0": {
+        "end": {
+          "column": 23,
+          "line": 1,
+        },
+        "start": {
+          "column": 0,
+          "line": 1,
+        },
+      },
+      "1": {
+        "end": {
+          "column": 51,
+          "line": 2,
+        },
+        "start": {
+          "column": 0,
+          "line": 2,
+        },
+      },
+      "2": {
+        "end": {
+          "column": 1,
+          "line": 3,
+        },
+        "start": {
+          "column": 0,
+          "line": 3,
+        },
+      },
+      "3": {
+        "end": {
+          "column": 0,
+          "line": 4,
+        },
+        "start": {
+          "column": 0,
+          "line": 4,
+        },
+      },
+      "4": {
+        "end": {
+          "column": 37,
+          "line": 5,
+        },
+        "start": {
+          "column": 0,
+          "line": 5,
+        },
+      },
+      "5": {
+        "end": {
+          "column": 14,
+          "line": 6,
+        },
+        "start": {
+          "column": 0,
+          "line": 6,
+        },
+      },
+      "6": {
+        "end": {
+          "column": 1,
+          "line": 7,
+        },
+        "start": {
+          "column": 0,
+          "line": 7,
+        },
+      },
+    },
+  },
   "<process-cwd>/src/function-count.ts": {
     "all": false,
     "b": {

test/coverage-test/coverage-report-tests/changed.test.ts

@@ -0,0 +1,22 @@
+import { expect, test } from 'vitest'
+import libCoverage from 'istanbul-lib-coverage'
+
+import { readCoverageJson } from './utils'
+
+test('report contains only the changed files', async () => {
+  const coverageJson = await readCoverageJson('./coverage/coverage-final.json')
+  const coverageMap = libCoverage.createCoverageMap(coverageJson as any)
+
+  expect(coverageMap.files()).toMatchInlineSnapshot(`
+    [
+      "<process-cwd>/src/file-to-change.ts",
+      "<process-cwd>/src/new-uncovered-file.ts",
+    ]
+  `)
+
+  const uncoveredFile = coverageMap.fileCoverageFor('<process-cwd>/src/new-uncovered-file.ts').toSummary()
+  expect(uncoveredFile.lines.pct).toBe(0)
+
+  const changedFile = coverageMap.fileCoverageFor('<process-cwd>/src/file-to-change.ts').toSummary()
+  expect(changedFile.lines.pct).toBeGreaterThanOrEqual(50)
+})

test/coverage-test/coverage-report-tests/utils.ts

@@ -17,8 +17,8 @@ interface CoverageFinalJson {
  * Read JSON coverage report from file system.
  * Normalizes paths to keep contents consistent between OS's
  */
-export async function readCoverageJson() {
-  const jsonReport = JSON.parse(readFileSync('./coverage/custom-json-report-name.json', 'utf8')) as CoverageFinalJson
+export async function readCoverageJson(name = './coverage/custom-json-report-name.json') {
+  const jsonReport = JSON.parse(readFileSync(name, 'utf8')) as CoverageFinalJson
 
   const normalizedReport: CoverageFinalJson['default'] = {}
 

test/coverage-test/option-tests/changed.test.ts

@@ -0,0 +1,6 @@
+import { test } from 'vitest'
+import { run } from '../src/file-to-change'
+
+test('test case for changed file', () => {
+  run()
+})

test/coverage-test/src/file-to-change.ts

@@ -0,0 +1,7 @@
+export function run() {
+  return 'This file will be modified by test cases'
+}
+
+export function uncoveredFunction() {
+  return 1 + 2
+}

test/coverage-test/testing-options.mjs

@@ -1,6 +1,13 @@
+import { readFileSync, rmSync, writeFileSync } from 'node:fs'
 import { startVitest } from 'vitest/node'
 
-/** @type {Record<string, Partial<import('vitest/config').UserConfig['test']>>[]} */
+/**
+ * @typedef {NonNullable<import('vitest/config').UserConfig['test']>} Config
+ * @typedef { () => void | Promise<void> } Callback
+ * @typedef {{ testConfig: Config, assertionConfig?: Config, after?: Callback, before?: Callback }} TestCase
+ */
+
+/** @type {TestCase[]} */
 const testCases = [
   {
     testConfig: {
@@ -51,10 +58,44 @@ const testCases = [
     },
     assertionConfig: null,
   },
+  {
+    testConfig: {
+      name: 'changed',
+      changed: 'HEAD',
+      coverage: {
+        include: ['src'],
+        reporter: 'json',
+        all: true,
+      },
+    },
+    assertionConfig: {
+      include: ['coverage-report-tests/changed.test.ts'],
+    },
+    before: () => {
+      let content = readFileSync('./src/file-to-change.ts', 'utf8')
+      content = content.replace('This file will be modified by test cases', 'Changed!')
+      writeFileSync('./src/file-to-change.ts', content, 'utf8')
+
+      writeFileSync('./src/new-uncovered-file.ts', `
+      // This file is not covered by any tests but should be picked by --changed
+      export default function helloworld() {
+        return 'Hello world'
+      }
+      `.trim(), 'utf8')
+    },
+    after: () => {
+      let content = readFileSync('./src/file-to-change.ts', 'utf8')
+      content = content.replace('Changed!', 'This file will be modified by test cases')
+      writeFileSync('./src/file-to-change.ts', content, 'utf8')
+      rmSync('./src/new-uncovered-file.ts')
+    },
+  },
 ]
 
 for (const provider of ['v8', 'istanbul']) {
-  for (const { testConfig, assertionConfig } of testCases) {
+  for (const { after, before, testConfig, assertionConfig } of testCases) {
+    await before?.()
+
     // Run test case
     await startVitest('test', ['option-tests/'], {
       config: false,
@@ -65,23 +106,23 @@ for (const provider of ['v8', 'istanbul']) {
         enabled: true,
         clean: true,
         all: false,
+        reporter: [],
         provider,
         ...testConfig.coverage,
       },
     })
 
-    checkExit()
-
-    if (!assertionConfig)
-      continue
-
     // Check generated coverage report
-    await startVitest('test', ['coverage-report-tests'], {
-      config: false,
-      watch: false,
-      ...assertionConfig,
-      name: `${provider} - assert ${testConfig.name}`,
-    })
+    if (assertionConfig) {
+      await startVitest('test', ['coverage-report-tests'], {
+        config: false,
+        watch: false,
+        ...assertionConfig,
+        name: `${provider} - assert ${testConfig.name}`,
+      })
+    }
+
+    await after?.()
 
     checkExit()
   }