feat(reporter api): unify reporter api with mutation-testing-elements (…

…#2798)

This change brings the `Reporter` API more in line with the schema of [`mutation-testing-elements`](https://github.com/stryker-mutator/mutation-testing-elements). It also adds the metrics result to the `onMutationTestReportReady` for convenience.

| Old | New |
|-|-|
| `MutantStatus.TimedOut` | `MutantStatus.Timeout` |
| `mutant.coveredByTests` (boolean) | `mutant.coveredBy` (array, an empty array means not covered) |
| `mutant.testFilter` (array) | `mutant.coveredBy` |
| `mutant.testFilter` (undefined) | `mutant.static` (true/false) |
| `mutant.runAllTests` (boolean) | `mutant.static` (true/false) |
| `mutant.errorMessage`, `mutant.ignoreReason` | `mutant.statusReason` |
| `mutant.nrOfTestsRan` | `mutant.testsCompleted` |
| `mutant.id` (number) | `mutant.id` (string) |
| `MatchedMutant` | `MutantTestCoverage` (name change) |

This has also a small impact on the `TestRunner` API, because the `Mutant` is used there as well.

BREAKING CHANGE: Changes to `Reporter` and `TestRunner` plugin API of Stryker

Fixes #2766

e2e/package.json

@@ -31,7 +31,7 @@
     "link-parent-bin": "^2.0.0",
     "load-grunt-tasks": "~5.1.0",
     "mocha": "~8.2.0",
-    "mutation-testing-metrics": "~1.5.2",
+    "mutation-testing-metrics": "~1.6.2",
     "rxjs": "~6.5.3",
     "semver": "~6.3.0",
     "ts-jest": "~26.3.0",

e2e/test/coverage-analysis/verify/verify.ts

@@ -59,7 +59,7 @@ describe('Coverage analysis', () => {
       const stryker = new Stryker(strykerOptions);
 
       // Act
-      const testsRan = (await stryker.runMutationTest()).reduce((a, b) => a + b.nrOfTestsRan, 0);
+      const testsRan = (await stryker.runMutationTest()).reduce((a, b) =>  a + (b.testsCompleted ?? 0), 0);
 
       // Assert
       const metricsResult = calculateMetrics(CoverageAnalysisReporter.instance?.report.files);
@@ -79,7 +79,7 @@ describe('Coverage analysis', () => {
       const stryker = new Stryker(strykerOptions);
 
       // Act
-      const testsRan = (await stryker.runMutationTest()).reduce((a, b) => a + b.nrOfTestsRan, 0);
+      const testsRan = (await stryker.runMutationTest()).reduce((a, b) => a + (b.testsCompleted ?? 0), 0);
 
       // Assert
       const metricsResult = calculateMetrics(CoverageAnalysisReporter.instance?.report.files);
@@ -99,7 +99,7 @@ describe('Coverage analysis', () => {
       const stryker = new Stryker(strykerOptions);
 
       // Act
-      const testsRan = (await stryker.runMutationTest()).reduce((a, b) => a + b.nrOfTestsRan, 0);
+      const testsRan = (await stryker.runMutationTest()).reduce((a, b) => a + (b.testsCompleted ?? 0), 0);
 
       // Assert
       const metricsResult = calculateMetrics(CoverageAnalysisReporter.instance?.report.files);

e2e/test/karma-webpack-with-ts/package.json

@@ -8,8 +8,8 @@
   },
   "dependencies": {
     "lit-element": "~2.3.1",
-    "mutation-testing-metrics": "~1.5.2",
-    "mutation-testing-report-schema": "~1.5.2"
+    "mutation-testing-metrics": "~1.6.2",
+    "mutation-testing-report-schema": "~1.6.0"
   },
   "devDependencies": {
     "@types/webpack-env": "~1.15.2"

packages/api/package.json

@@ -38,7 +38,8 @@
     "node": ">=10"
   },
   "dependencies": {
-    "mutation-testing-report-schema": "~1.5.2",
+    "mutation-testing-report-schema": "~1.6.0",
+    "mutation-testing-metrics": "~1.6.2",
     "surrial": "~2.0.2",
     "tslib": "~2.1.0"
   },

packages/api/src/core/index.ts

@@ -2,10 +2,15 @@ export { File } from './file';
 export { Position } from './position';
 export { Location } from './location';
 export { Range } from './range';
-export { Mutant } from './mutant';
+export * from './mutant';
 export * from '../../src-generated/stryker-core';
 export * from './report-types';
 export * from './stryker-options-schema';
 export * from './partial-stryker-options';
 export * from './instrument';
 export * from './mutant-coverage';
+
+/**
+ * Re-export all members from "mutation-testing-report-schema" under the `schema` key
+ */
+export * as schema from 'mutation-testing-report-schema';

packages/api/src/core/instrument.ts

@@ -12,7 +12,7 @@ export const INSTRUMENTER_CONSTANTS = Object.freeze({
 } as const);
 
 export interface InstrumenterContext {
-  activeMutant?: number;
+  activeMutant?: string;
   currentTestId?: string;
   mutantCoverage?: MutantCoverage;
 }

packages/api/src/core/mutant-coverage.ts

@@ -5,4 +5,7 @@ export interface MutantCoverage {
 
 export type CoveragePerTestId = Record<string, CoverageData>;
 
-export type CoverageData = Record<number, number>;
+/**
+ * Keys are mutant ids, the numbers are the amount of times it was hit.
+ */
+export type CoverageData = Record<string, number>;

packages/api/src/core/mutant.ts

@@ -1,36 +1,43 @@
+import * as schema from 'mutation-testing-report-schema';
+
 import { Range } from './range';
-import { Location } from './location';
+
+export { MutantStatus } from 'mutation-testing-report-schema';
+
+// We're reusing the `MutantResult` interface here to acquire uniformity.
 
 /**
- * Represents a mutant
+ * Represents a mutant in its initial state.
  */
-export interface Mutant {
-  /**
-   * The id of the mutant. Unique within a run.
-   */
-  id: number;
-  /**
-   * The name of the mutator that generated this mutant.
-   */
-  mutatorName: string;
+export interface Mutant extends Pick<schema.MutantResult, 'id' | 'location' | 'mutatorName' | 'replacement' | 'statusReason'> {
   /**
    * The file name from which this mutant originated
    */
   fileName: string;
   /**
    * The range of this mutant (from/to within the file)
+   * deprecate?
    */
   range: Range;
   /**
-   * The line number/column location of this mutant
-   */
-  location: Location;
-  /**
-   * The replacement (actual mutated code)
+   * Actual mutation that has been applied.
    */
   replacement: string;
   /**
-   * If the mutant was ignored during generation, the reason for ignoring it, otherwise `undefined`
+   * The status if a mutant if known. This should be undefined for a mutant that still needs testing.
    */
-  ignoreReason?: string;
+  status?: schema.MutantStatus;
 }
+
+/**
+ * Represents a mutant in its matched-with-the-tests state, ready to be tested.
+ */
+export type MutantTestCoverage = Mutant &
+  Pick<schema.MutantResult, 'coveredBy' | 'static'> & {
+    estimatedNetTime: number;
+  };
+
+/**
+ * Represents a mutant in its final state, ready to be reported.
+ */
+export type MutantResult = Mutant & schema.MutantResult;

packages/api/src/report/index.ts

@@ -1,14 +1,2 @@
-import * as mutationTestReportSchema from 'mutation-testing-report-schema';
-
 export { Reporter } from './reporter';
-export * from './mutant-result';
-export { MutantStatus } from './mutant-status';
 export { SourceFile } from './source-file';
-export { MatchedMutant } from './matched-mutant';
-export {
-  /**
-   * Types exported directly from mutation-testing-schema
-   * @see https://github.com/stryker-mutator/mutation-testing-elements/tree/master/packages/mutation-testing-report-schema
-   */
-  mutationTestReportSchema,
-};