feat(jest): report test files and test positions (#2808)

Report the test file names and test positions from the jest-runner plugin back to the main process. They will end up in your JSON report.

e2e/package.json

@@ -19,7 +19,7 @@
     "grunt": "~1.0.4",
     "jasmine": "~3.6.2",
     "jasmine-core": "~3.6.0",
-    "jest": "~26.4.2",
+    "jest": "~26.6.1",
     "jest-environment-jsdom-sixteen": "^1.0.3",
     "karma": "~6.0.1",
     "karma-chai": "~0.1.0",

e2e/test/jest-node/stryker.conf.json

@@ -9,6 +9,7 @@
     "clear-text",
     "progress",
     "html",
-    "event-recorder"
+    "event-recorder",
+    "json"
   ]
 }

e2e/test/jest-node/verify/verify.ts

@@ -1,4 +1,7 @@
-
+import { expect } from 'chai';
+import path from 'path';
+import { promises as fsPromises } from 'fs';
+import { MutationTestResult, TestFileDefinitionDictionary } from 'mutation-testing-report-schema';
 import { expectMetricsResult, produceMetrics } from '../../../helpers';
 
 describe('After running stryker with test runner jest on test environment "node"', () => {
@@ -18,4 +21,23 @@ describe('After running stryker with test runner jest on test environment "node"
       })
     });
   });
+
+  it('should report test files and test locations', async () => {
+    const report: MutationTestResult = require('../reports/mutation/mutation.json');
+    const expectedTestFiles: TestFileDefinitionDictionary = {
+      [path.resolve(__dirname, '..', 'src', 'sum.test.js')]: {
+        source: await fsPromises.readFile(path.resolve(__dirname, '..', 'src', 'sum.test.js'), 'utf-8'),
+        tests: [{
+          id: '0',
+          name: 'adds 1 + 2 to equal 3',
+          location: { start: { line: 3, column: 2 } } // columns should be 1, but for some reason aren't. Best guess if jest? 🤷‍♂️
+        }, {
+          id: '1',
+          name: 'sub 1 - 0 to equal 1',
+          location: { start: { line: 6, column: 2 } }
+        }]
+      }
+    };
+    expect(report.testFiles).deep.eq(expectedTestFiles);
+  });
 });

packages/core/src/config/file-matcher.ts

@@ -0,0 +1,22 @@
+import path from 'path';
+
+import minimatch from 'minimatch';
+
+/**
+ * A helper class for matching files using the `disableTypeChecks` setting.
+ */
+export class FileMatcher {
+  private readonly pattern: string | false;
+
+  constructor(pattern: string | false) {
+    if (pattern !== false) {
+      this.pattern = path.resolve(pattern);
+    } else {
+      this.pattern = pattern;
+    }
+  }
+
+  public matches(fileName: string): boolean {
+    return !!this.pattern && minimatch(path.resolve(fileName), this.pattern);
+  }
+}

packages/core/src/config/index.ts

@@ -1,3 +1,4 @@
 export * from './read-config';
 export * from './options-validator';
 export * from './build-schema-with-plugin-contributions';
+export * from './file-matcher';

packages/core/src/process/3-dry-run-executor.ts

@@ -27,6 +27,7 @@ import { ConfigError } from '../errors';
 import { findMutantTestCoverage } from '../mutants';
 import { Pool, createTestRunnerPool } from '../concurrent/pool';
 import { ConcurrencyTokenProvider } from '../concurrent';
+import { FileMatcher } from '../config';
 
 import { MutationTestContext } from './4-mutation-test-executor';
 import { MutantInstrumenterContext } from './2-mutant-instrumenter-executor';
@@ -141,9 +142,17 @@ export class DryRunExecutor {
    * @param dryRunResult the completed result
    */
   private remapSandboxFilesToOriginalFiles(dryRunResult: CompleteDryRunResult) {
+    const disableTypeCheckingFileMatcher = new FileMatcher(this.options.disableTypeChecks);
     dryRunResult.tests.forEach((test) => {
       if (test.fileName) {
         test.fileName = this.sandbox.originalFileFor(test.fileName);
+
+        // HACK line numbers of the tests can be offset by 1 because the disable type checks preprocessor could have added a `// @ts-nocheck` line.
+        // We correct for that here if needed
+        // If we do more complex stuff in sandbox preprocessing in the future, we might want to add a robust remapping logic
+        if (test.startPosition && disableTypeCheckingFileMatcher.matches(test.fileName)) {
+          test.startPosition.line--;
+        }
       }
     });
   }

packages/core/src/reporters/mutation-test-report-helper.ts

@@ -203,6 +203,7 @@ export class MutationTestReportHelper {
     return {
       id: remapTestId(test.id),
       name: test.name,
+      location: test.startPosition ? { start: this.toPosition(test.startPosition) } : undefined,
     };
   }
 

packages/core/src/sandbox/disable-type-checks-preprocessor.ts

@@ -1,6 +1,5 @@
 import path from 'path';
 
-import minimatch from 'minimatch';
 import { commonTokens, tokens } from '@stryker-mutator/api/plugin';
 import { File, StrykerOptions } from '@stryker-mutator/api/core';
 import type { disableTypeChecks } from '@stryker-mutator/instrumenter';
@@ -9,6 +8,7 @@ import { propertyPath, PropertyPathBuilder } from '@stryker-mutator/util';
 
 import { coreTokens } from '../di';
 import { isWarningEnabled } from '../utils/object-utils';
+import { FileMatcher } from '../config/file-matcher';
 
 import { FilePreprocessor } from './file-preprocessor';
 
@@ -21,41 +21,35 @@ export class DisableTypeChecksPreprocessor implements FilePreprocessor {
   constructor(private readonly log: Logger, private readonly options: StrykerOptions, private readonly impl: typeof disableTypeChecks) {}
 
   public async preprocess(files: File[]): Promise<File[]> {
-    if (this.options.disableTypeChecks === false) {
-      return files;
-    } else {
-      const pattern = path.resolve(this.options.disableTypeChecks);
-      let warningLogged = false;
-      const outFiles = await Promise.all(
-        files.map(async (file) => {
-          if (minimatch(path.resolve(file.name), pattern)) {
-            try {
-              return await this.impl(file, { plugins: this.options.mutator.plugins });
-            } catch (err) {
-              if (isWarningEnabled('preprocessorErrors', this.options.warnings)) {
-                warningLogged = true;
-                this.log.warn(
-                  `Unable to disable type checking for file "${
-                    file.name
-                  }". Shouldn't type checking be disabled for this file? Consider configuring a more restrictive "${propertyPath<StrykerOptions>(
-                    'disableTypeChecks'
-                  )}" settings (or turn it completely off with \`false\`)`,
-                  err
-                );
-              }
-              return file;
+    const matcher = new FileMatcher(this.options.disableTypeChecks);
+    let warningLogged = false;
+    const outFiles = await Promise.all(
+      files.map(async (file) => {
+        if (matcher.matches(path.resolve(file.name))) {
+          try {
+            return await this.impl(file, { plugins: this.options.mutator.plugins });
+          } catch (err) {
+            if (isWarningEnabled('preprocessorErrors', this.options.warnings)) {
+              warningLogged = true;
+              this.log.warn(
+                `Unable to disable type checking for file "${
+                  file.name
+                }". Shouldn't type checking be disabled for this file? Consider configuring a more restrictive "${propertyPath<StrykerOptions>(
+                  'disableTypeChecks'
+                )}" settings (or turn it completely off with \`false\`)`,
+                err
+              );
             }
-          } else {
             return file;
           }
-        })
-      );
-      if (warningLogged) {
-        this.log.warn(
-          `(disable "${PropertyPathBuilder.create<StrykerOptions>().prop('warnings').prop('preprocessorErrors')}" to ignore this warning`
-        );
-      }
-      return outFiles;
+        } else {
+          return file;
+        }
+      })
+    );
+    if (warningLogged) {
+      this.log.warn(`(disable "${PropertyPathBuilder.create<StrykerOptions>().prop('warnings').prop('preprocessorErrors')}" to ignore this warning`);
     }
+    return outFiles;
   }
 }

packages/core/test/unit/config/file-matcher.spec.ts

@@ -0,0 +1,24 @@
+import { expect } from 'chai';
+
+import { FileMatcher } from '../../../src/config';
+
+describe(FileMatcher.name, () => {
+  describe(FileMatcher.prototype.matches.name, () => {
+    it('should match when the glob pattern matches', () => {
+      const sut = new FileMatcher('src/**/*.ts');
+      expect(sut.matches('src/foo.ts')).true;
+    });
+
+    it('should not match if the pattern is set to `false`', () => {
+      const sut = new FileMatcher(false);
+      expect(sut.matches('src/foo.js')).false;
+    });
+
+    it("should not match if the glob pattern doesn't match", () => {
+      const sut = new FileMatcher('src/**/*.js');
+      expect(sut.matches('test/foo.spec.js')).false;
+    });
+
+    // more tests would test the internals of minimatch itself. We expect that to work.
+  });
+});

packages/core/test/unit/process/3-dry-run-executor.spec.ts

@@ -144,6 +144,24 @@ describe(DryRunExecutor.name, () => {
         expect(sandbox.originalFileFor).calledWith('.stryker-tmp/sandbox-123/test/foo.spec.js');
       });
 
+      it('should remap test locations when type checking was disabled for a test file', async () => {
+        runResult.tests.push(
+          factory.successTestResult({ fileName: '.stryker-tmp/sandbox-123/test/foo.spec.js', startPosition: { line: 3, column: 1 } }),
+          factory.successTestResult({ fileName: '.stryker-tmp/sandbox-123/testResources/foo.spec.js', startPosition: { line: 5, column: 1 } })
+        );
+        sandbox.originalFileFor
+          .withArgs('.stryker-tmp/sandbox-123/test/foo.spec.js')
+          .returns('test/foo.spec.js')
+          .withArgs('.stryker-tmp/sandbox-123/testResources/foo.spec.js')
+          .returns('testResources/foo.spec.js');
+        await sut.execute();
+        const actualDryRunResult = injectorMock.provideValue.getCalls().find((call) => call.args[0] === coreTokens.dryRunResult)!
+          .args[1] as DryRunResult;
+        assertions.expectCompleted(actualDryRunResult);
+        expect(actualDryRunResult.tests[0].startPosition).deep.eq({ line: 2, column: 1 });
+        expect(actualDryRunResult.tests[1].startPosition).deep.eq({ line: 5, column: 1 }); // should not have been remapped, since type checking wasn't disabled here
+      });
+
       it('should have logged the amount of tests ran', async () => {
         runResult.tests.push(factory.successTestResult({ timeSpentMs: 10 }));
         runResult.tests.push(factory.successTestResult({ timeSpentMs: 10 }));