feat(report): add test details and metadata to JSON report (#2755)

* Update test runner api:\
  Add `fileName` and `startPosition` to the `BaseTestResult`. This makes it possible to correlate tests with mutants inside the mutation test report in the future.
* Update JSON report:\
  Add `testFiles`, `framework`, `projectRoot`, and `config` to the JSON report and by extension also to the HTML report, although they won't be visible yet.

packages/api/src/test-runner/test-result.ts

@@ -1,3 +1,5 @@
+import { Position } from '../core';
+
 import { TestStatus } from './test-status';
 
 /**
@@ -16,6 +18,16 @@ export interface BaseTestResult {
    * The time it took to run the test
    */
   timeSpentMs: number;
+
+  /**
+   * The file where this test was defined in (if known)
+   */
+  fileName?: string;
+
+  /**
+   * The position of the test (if known)
+   */
+  startPosition?: Position;
 }
 
 export interface FailedTestResult extends BaseTestResult {

packages/core/src/process/3-dry-run-executor.ts

@@ -70,15 +70,17 @@ export class DryRunExecutor {
     commonTokens.logger,
     commonTokens.options,
     coreTokens.timer,
-    coreTokens.concurrencyTokenProvider
+    coreTokens.concurrencyTokenProvider,
+    coreTokens.sandbox
   );
 
   constructor(
     private readonly injector: Injector<DryRunContext>,
     private readonly log: Logger,
     private readonly options: StrykerOptions,
     private readonly timer: I<Timer>,
-    private readonly concurrencyTokenProvider: I<ConcurrencyTokenProvider>
+    private readonly concurrencyTokenProvider: I<ConcurrencyTokenProvider>,
+    public readonly sandbox: I<Sandbox>
   ) {}
 
   public async execute(): Promise<Injector<MutationTestContext>> {
@@ -128,10 +130,24 @@ export class DryRunExecutor {
     const grossTimeMS = this.timer.elapsedMs(INITIAL_TEST_RUN_MARKER);
     const humanReadableTimeElapsed = this.timer.humanReadableElapsed(INITIAL_TEST_RUN_MARKER);
     this.validateResultCompleted(dryRunResult);
+
+    this.remapSandboxFilesToOriginalFiles(dryRunResult);
     const timing = this.calculateTiming(grossTimeMS, humanReadableTimeElapsed, dryRunResult.tests);
     return { dryRunResult, timing };
   }
 
+  /**
+   * Remaps test files to their respective original names outside the sandbox.
+   * @param dryRunResult the completed result
+   */
+  private remapSandboxFilesToOriginalFiles(dryRunResult: CompleteDryRunResult) {
+    dryRunResult.tests.forEach((test) => {
+      if (test.fileName) {
+        test.fileName = this.sandbox.originalFileFor(test.fileName);
+      }
+    });
+  }
+
   private logInitialTestRunSucceeded(tests: TestResult[], timing: Timing) {
     this.log.info(
       'Initial test run succeeded. Ran %s tests in %s (net %s ms, overhead %s ms).',

packages/core/src/reporters/mutation-test-report-helper.ts

@@ -4,7 +4,7 @@ import { Location, Position, StrykerOptions, Mutant, MutantTestCoverage, MutantR
 import { Logger } from '@stryker-mutator/api/logging';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
 import { Reporter } from '@stryker-mutator/api/report';
-import { normalizeWhitespaces } from '@stryker-mutator/util';
+import { normalizeWhitespaces, requireResolve } from '@stryker-mutator/util';
 import { calculateMutationTestMetrics, MutationTestMetricsResult } from 'mutation-testing-metrics';
 import { CompleteDryRunResult, MutantRunResult, MutantRunStatus, TestResult } from '@stryker-mutator/api/test-runner';
 import { CheckStatus, PassedCheckResult, CheckResult } from '@stryker-mutator/api/check';
@@ -13,6 +13,17 @@ import { coreTokens } from '../di';
 import { InputFileCollection } from '../input/input-file-collection';
 import { setExitCode } from '../utils/object-utils';
 
+const STRYKER_FRAMEWORK: Readonly<Pick<schema.FrameworkInformation, 'branding' | 'name' | 'version'>> = Object.freeze({
+  name: 'StrykerJS',
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  version: require('../../../package.json').version,
+  branding: {
+    homepageUrl: 'https://stryker-mutator.io',
+    imageUrl:
+      "data:image/svg+xml;utf8,%3Csvg viewBox='0 0 1458 1458' xmlns='http://www.w3.org/2000/svg' fill-rule='evenodd' clip-rule='evenodd' stroke-linejoin='round' stroke-miterlimit='2'%3E%3Cpath fill='none' d='M0 0h1458v1458H0z'/%3E%3CclipPath id='a'%3E%3Cpath d='M0 0h1458v1458H0z'/%3E%3C/clipPath%3E%3Cg clip-path='url(%23a)'%3E%3Cpath d='M1458 729c0 402.655-326.345 729-729 729S0 1131.655 0 729C0 326.445 326.345 0 729 0s729 326.345 729 729' fill='%23e74c3c' fill-rule='nonzero'/%3E%3Cpath d='M778.349 1456.15L576.6 1254.401l233-105 85-78.668v-64.332l-257-257-44-187-50-208 251.806-82.793L1076.6 389.401l380.14 379.15c-19.681 367.728-311.914 663.049-678.391 687.599z' fill-opacity='.3'/%3E%3Cpath d='M753.4 329.503c41.79 0 74.579 7.83 97.925 25.444 23.571 18.015 41.69 43.956 55.167 77.097l11.662 28.679 165.733-58.183-14.137-32.13c-26.688-60.655-64.896-108.61-114.191-144.011-49.329-35.423-117.458-54.302-204.859-54.302-50.78 0-95.646 7.376-134.767 21.542-40.093 14.671-74.09 34.79-102.239 60.259-28.84 26.207-50.646 57.06-65.496 92.701-14.718 35.052-22.101 72.538-22.101 112.401 0 72.536 20.667 133.294 61.165 182.704 38.624 47.255 98.346 88.037 179.861 121.291 42.257 17.475 78.715 33.125 109.227 46.994 27.193 12.361 49.294 26.124 66.157 41.751 15.309 14.186 26.497 30.584 33.63 49.258 7.721 20.214 11.16 45.69 11.16 76.402 0 28.021-4.251 51.787-13.591 71.219-8.832 18.374-20.171 33.178-34.523 44.219-14.787 11.374-31.193 19.591-49.393 24.466-19.68 5.359-39.14 7.993-58.69 7.993-29.359 0-54.387-3.407-75.182-10.747-20.112-7.013-37.144-16.144-51.259-27.486-13.618-11.009-24.971-23.766-33.744-38.279-9.64-15.8-17.272-31.924-23.032-48.408l-10.965-31.376-161.669 60.585 10.734 30.124c10.191 28.601 24.197 56.228 42.059 82.748 18.208 27.144 41.322 51.369 69.525 72.745 27.695 21.075 60.904 38.218 99.481 51.041 37.777 12.664 82.004 19.159 132.552 19.159 49.998 0 95.818-8.321 137.611-24.622 42.228-16.471 78.436-38.992 108.835-67.291 30.719-28.597 54.631-62.103 71.834-100.642 17.263-38.56 25.923-79.392 25.923-122.248 0-54.339-8.368-100.37-24.208-138.32-16.29-38.759-38.252-71.661-65.948-98.797-26.965-26.418-58.269-48.835-93.858-67.175-33.655-17.241-69.196-33.11-106.593-47.533-35.934-13.429-65.822-26.601-89.948-39.525-22.153-11.868-40.009-24.21-53.547-37.309-11.429-11.13-19.83-23.678-24.718-37.664-5.413-15.49-7.98-33.423-7.98-53.577 0-40.883 11.293-71.522 37.086-90.539 28.443-20.825 64.985-30.658 109.311-30.658z' fill='%23f1c40f' fill-rule='nonzero'/%3E%3Cpath d='M720 0h18v113h-18zM1458 738v-18h-113v18h113zM720 1345h18v113h-18zM113 738v-18H0v18h113z'/%3E%3C/g%3E%3C/svg%3E",
+  },
+});
+
 /**
  * A helper class to convert and report mutants that survived or get killed
  */
@@ -124,45 +135,68 @@ export class MutationTestReportHelper {
       schemaVersion: '1.0',
       thresholds: this.options.thresholds,
       testFiles: this.toTestFiles(remapTestId),
+      projectRoot: process.cwd(),
+      config: this.options,
+      framework: {
+        ...STRYKER_FRAMEWORK,
+        dependencies: this.discoverDependencies(),
+      },
     };
   }
 
   private toFileResults(
     results: readonly MutantResult[],
     remapTestIds: (ids: string[] | undefined) => string[] | undefined
   ): schema.FileResultDictionary {
-    const resultDictionary: schema.FileResultDictionary = Object.create(null);
-
-    results.forEach((mutantResult) => {
-      const fileResult = resultDictionary[mutantResult.fileName];
-      const mutant = this.toMutantResult(mutantResult, remapTestIds);
-      if (fileResult) {
-        fileResult.mutants.push(mutant);
-      } else {
-        const sourceFile = this.inputFiles.files.find((file) => file.name === mutantResult.fileName);
-        if (sourceFile) {
-          resultDictionary[mutantResult.fileName] = {
-            language: this.determineLanguage(sourceFile.name),
-            mutants: [mutant],
-            source: sourceFile.textContent,
-          };
-        } else {
-          this.log.warn(
-            normalizeWhitespaces(`File "${mutantResult.fileName}" not found
-          in input files, but did receive mutant result for it. This shouldn't happen`)
-          );
-        }
-      }
-    });
-    return resultDictionary;
+    return results.reduce<schema.FileResultDictionary>((acc, mutantResult) => {
+      const fileResult = acc[mutantResult.fileName] ?? (acc[mutantResult.fileName] = this.toFileResult(mutantResult.fileName));
+      fileResult.mutants.push(this.toMutantResult(mutantResult, remapTestIds));
+      return acc;
+    }, Object.create(null));
   }
 
   private toTestFiles(remapTestId: (id: string) => string): schema.TestFileDefinitionDictionary {
-    return {
-      '': {
-        tests: this.dryRunResult.tests.map((test) => this.toTestDefinition(test, remapTestId)),
-      },
+    return this.dryRunResult.tests.reduce<schema.TestFileDefinitionDictionary>((acc, testResult) => {
+      const test = this.toTestDefinition(testResult, remapTestId);
+      const fileName = testResult.fileName ?? ''; // by default we accumulate tests under the '' key
+      const testFile = acc[fileName] ?? (acc[fileName] = this.toTestFile(fileName));
+      testFile.tests.push(test);
+      return acc;
+    }, Object.create(null));
+  }
+
+  private toFileResult(fileName: string): schema.FileResult {
+    const fileResult: schema.FileResult = {
+      language: this.determineLanguage(fileName),
+      mutants: [],
+      source: '',
     };
+    const sourceFile = this.inputFiles.files.find((file) => file.name === fileName);
+    if (sourceFile) {
+      fileResult.source = sourceFile.textContent;
+    } else {
+      this.log.warn(
+        normalizeWhitespaces(`File "${fileName}" not found
+    in input files, but did receive mutant result for it. This shouldn't happen`)
+      );
+    }
+    return fileResult;
+  }
+
+  private toTestFile(fileName: string | undefined): schema.TestFile {
+    const testFile: schema.TestFile = { tests: [] };
+    if (fileName) {
+      const sourceFile = this.inputFiles.files.find((file) => file.name === fileName);
+      if (sourceFile) {
+        testFile.source = sourceFile.textContent;
+      } else {
+        this.log.warn(
+          normalizeWhitespaces(`Test file "${fileName}" not found
+        in input files, but did receive test result for it. This shouldn't happen.`)
+        );
+      }
+    }
+    return testFile;
   }
 
   private toTestDefinition(test: TestResult, remapTestId: (id: string) => string): schema.TestDefinition {
@@ -209,4 +243,43 @@ export class MutationTestReportHelper {
       line: pos.line + 1,
     };
   }
+
+  private discoverDependencies(): schema.Dependencies {
+    const discover = (specifier: string) => {
+      try {
+        return [specifier, (requireResolve(`${specifier}/package.json`) as { version: string }).version];
+      } catch {
+        // package does not exist...
+        return undefined;
+      }
+    };
+    const dependencies = [
+      '@stryker-mutator/mocha-runner',
+      '@stryker-mutator/karma-runner',
+      '@stryker-mutator/jasmine-runner',
+      '@stryker-mutator/jest-runner',
+      '@stryker-mutator/typescript-checker',
+      'karma',
+      'karma-chai',
+      'karma-chrome-launcher',
+      'karma-jasmine',
+      'karma-mocha',
+      'mocha',
+      'jasmine',
+      'jasmine-core',
+      'jest',
+      'react-scripts',
+      'typescript',
+      '@angular/cli',
+      'webpack',
+      'webpack-cli',
+      'ts-jest',
+    ];
+    return dependencies.map(discover).reduce<schema.Dependencies>((acc, dependency) => {
+      if (dependency) {
+        acc[dependency[0]] = dependency[1];
+      }
+      return acc;
+    }, Object.create(null));
+  }
 }

packages/core/src/sandbox/sandbox.ts

@@ -65,6 +65,10 @@ export class Sandbox implements Disposable {
     return sandboxFileName;
   }
 
+  public originalFileFor(sandboxFileName: string): string {
+    return path.resolve(sandboxFileName).replace(this.workingDirectory, process.cwd());
+  }
+
   private fillSandbox(): Promise<void[]> {
     return from(this.files)
       .pipe(

packages/core/test/unit/process/3-dry-run-executor.spec.ts

@@ -1,7 +1,7 @@
 import { EOL } from 'os';
 
 import { Injector } from 'typed-inject';
-import { factory, testInjector } from '@stryker-mutator/test-helpers';
+import { assertions, factory, testInjector } from '@stryker-mutator/test-helpers';
 import sinon from 'sinon';
 import { TestRunner, CompleteDryRunResult, ErrorDryRunResult, TimeoutDryRunResult, DryRunResult } from '@stryker-mutator/api/test-runner';
 import { expect } from 'chai';
@@ -16,6 +16,7 @@ import { coreTokens } from '../../../src/di';
 import { ConfigError } from '../../../src/errors';
 import { ConcurrencyTokenProvider, Pool } from '../../../src/concurrent';
 import { createTestRunnerPoolMock } from '../../helpers/producers';
+import { Sandbox } from '../../../src/sandbox';
 
 describe(DryRunExecutor.name, () => {
   let injectorMock: sinon.SinonStubbedInstance<Injector<MutationTestContext>>;
@@ -24,6 +25,7 @@ describe(DryRunExecutor.name, () => {
   let timerMock: sinon.SinonStubbedInstance<Timer>;
   let testRunnerMock: sinon.SinonStubbedInstance<Required<TestRunner>>;
   let concurrencyTokenProviderMock: sinon.SinonStubbedInstance<ConcurrencyTokenProvider>;
+  let sandbox: sinon.SinonStubbedInstance<Sandbox>;
 
   beforeEach(() => {
     timerMock = sinon.createStubInstance(Timer);
@@ -36,12 +38,14 @@ describe(DryRunExecutor.name, () => {
     concurrencyTokenProviderMock = sinon.createStubInstance(ConcurrencyTokenProvider);
     injectorMock = factory.injector();
     injectorMock.resolve.withArgs(coreTokens.testRunnerPool).returns(testRunnerPoolMock as I<Pool<TestRunner>>);
+    sandbox = sinon.createStubInstance(Sandbox);
     sut = new DryRunExecutor(
       injectorMock as Injector<DryRunContext>,
       testInjector.logger,
       testInjector.options,
       timerMock,
-      concurrencyTokenProviderMock
+      concurrencyTokenProviderMock,
+      sandbox
     );
   });
 
@@ -118,7 +122,7 @@ describe(DryRunExecutor.name, () => {
         expect(injector.provideValue).calledWithExactly(coreTokens.timeOverheadMS, 0);
       });
 
-      it('should provide the result', async () => {
+      it('should provide the dry run result', async () => {
         timerMock.elapsedMs.returns(42);
         runResult.tests.push(factory.successTestResult());
         runResult.mutantCoverage = {
@@ -129,6 +133,17 @@ describe(DryRunExecutor.name, () => {
         expect(actualInjector.provideValue).calledWithExactly(coreTokens.dryRunResult, runResult);
       });
 
+      it('should remap test files that are reported', async () => {
+        runResult.tests.push(factory.successTestResult({ fileName: '.stryker-tmp/sandbox-123/test/foo.spec.js' }));
+        sandbox.originalFileFor.returns('test/foo.spec.js');
+        await sut.execute();
+        const actualDryRunResult = injectorMock.provideValue.getCalls().find((call) => call.args[0] === coreTokens.dryRunResult)!
+          .args[1] as DryRunResult;
+        assertions.expectCompleted(actualDryRunResult);
+        expect(actualDryRunResult.tests[0].fileName).eq('test/foo.spec.js');
+        expect(sandbox.originalFileFor).calledWith('.stryker-tmp/sandbox-123/test/foo.spec.js');
+      });
+
       it('should have logged the amount of tests ran', async () => {
         runResult.tests.push(factory.successTestResult({ timeSpentMs: 10 }));
         runResult.tests.push(factory.successTestResult({ timeSpentMs: 10 }));