feat(mocha-runner): support mocha 8 (#2259)

* Support peer dependency range mocha@<9
* Add support for [rootHooks](mochajs/mocha#4244)
* Ignore `--parallel` flag (stryker will handle concurrency).

e2e/package.json

@@ -27,7 +27,7 @@
     "karma-webpack": "^4.0.2",
     "link-parent-bin": "~1.0.0",
     "load-grunt-tasks": "~5.1.0",
-    "mocha": "~7.1.1",
+    "mocha": "^8.0.1",
     "mutation-testing-metrics": "~1.3.0",
     "rxjs": "~6.5.3",
     "semver": "~6.3.0",

e2e/test/mocha-mocha/.mocharc.jsonc

@@ -0,0 +1,4 @@
+{
+    "require": "./test/helpers/testSetup.js",
+    "spec": ["test/unit/*.js"]
+}

e2e/test/mocha-mocha/stryker.conf.js

@@ -5,8 +5,5 @@ module.exports = function (config) {
     testRunner: 'mocha',
     reporters: ['clear-text', 'html', 'event-recorder'],
     maxConcurrentTestRunners: 2,
-    mochaOptions: {
-      spec: ['test/*.js', 'helpers/*.js']
-    },
   });
 };

e2e/test/mocha-mocha/test/helpers/testSetup.js

@@ -0,0 +1,5 @@
+exports.mochaHooks = {
+  beforeAll() {
+    global.expect = require('chai').expect;
+  }
+}

e2e/test/mocha-mocha/test/AddSpec.js → e2e/test/mocha-mocha/test/unit/AddSpec.js

@@ -1,5 +1,4 @@
-var expect = require('chai').expect;
-var addModule = require('../src/Add');
+var addModule = require('../../src/Add');
 var add = addModule.add;
 var addOne = addModule.addOne;
 var isNegativeNumber = addModule.isNegativeNumber;

e2e/test/mocha-mocha/test/CircleSpec.js → e2e/test/mocha-mocha/test/unit/CircleSpec.js

@@ -1,5 +1,4 @@
-var expect = require('chai').expect;
-var circleModule = require('../src/Circle');
+var circleModule = require('../../src/Circle');
 var getCircumference = circleModule.getCircumference;
 
 describe('Circle', function() {

package.json

@@ -33,7 +33,7 @@
     "json-schema-to-typescript": "~9.1.0",
     "lerna": "^3.10.7",
     "link-parent-bin": "~1.0.0",
-    "mocha": "^6.1.2",
+    "mocha": "^8.0.1",
     "nyc": "^15.0.0",
     "prettier": "2.0.5",
     "rimraf": "^3.0.0",

packages/mocha-runner/package.json

@@ -50,6 +50,6 @@
   },
   "peerDependencies": {
     "@stryker-mutator/core": "^3.0.0",
-    "mocha": ">= 2.3.3 < 8"
+    "mocha": ">= 2.3.3 < 9"
   }
 }

packages/mocha-runner/src/LibWrapper.ts

@@ -5,6 +5,8 @@ import { MochaOptions } from '../src-generated/mocha-runner-options';
 
 let loadOptions: undefined | ((argv?: string[] | string) => { [key: string]: any } | undefined);
 let handleFiles: undefined | ((options: MochaOptions) => string[]);
+let handleRequires: undefined | ((requires?: string[]) => Promise<any>);
+let loadRootHooks: undefined | ((rootHooks: any) => Promise<any>);
 
 try {
   /*
@@ -19,7 +21,11 @@ try {
 
 try {
   // https://github.com/mochajs/mocha/blob/master/lib/cli/run-helpers.js#L132
-  handleFiles = require('mocha/lib/cli/run-helpers').handleFiles;
+  const runHelpers = require('mocha/lib/cli/run-helpers');
+  handleFiles = runHelpers.handleFiles;
+  handleRequires = runHelpers.handleRequires; // handleRequires is available since mocha v7.2
+  loadRootHooks = runHelpers.loadRootHooks; // loadRootHooks is available since mocha v7.2
+
   if (!handleFiles) {
     // Might be moved: https://github.com/mochajs/mocha/commit/15b96afccaf508312445770e3af1c145d90b28c6#diff-39b692a81eb0c9f3614247af744ab4a8
     handleFiles = require('mocha/lib/cli/collect-files');
@@ -37,4 +43,6 @@ export default class LibWrapper {
   public static multimatch = multimatch;
   public static loadOptions = loadOptions;
   public static handleFiles = handleFiles;
+  public static handleRequires = handleRequires;
+  public static loadRootHooks = loadRootHooks;
 }

packages/mocha-runner/src/MochaTestRunner.ts

@@ -18,22 +18,23 @@ const DEFAULT_TEST_PATTERN = 'test/**/*.js';
 export class MochaTestRunner implements TestRunner {
   private testFileNames: string[];
   private readonly mochaOptions: MochaOptions;
+  private rootHooks: any;
 
   public static inject = tokens(commonTokens.logger, commonTokens.sandboxFileNames, commonTokens.options);
   constructor(private readonly log: Logger, private readonly allFileNames: readonly string[], options: StrykerOptions) {
     this.mochaOptions = (options as MochaRunnerWithStrykerOptions).mochaOptions;
-    this.additionalRequires();
     StrykerMochaReporter.log = log;
   }
 
-  public init(): void {
+  public async init(): Promise<void> {
     if (LibWrapper.handleFiles) {
       this.log.debug("Mocha >= 6 detected. Using mocha's `handleFiles` to load files");
       this.testFileNames = this.mocha6DiscoverFiles(LibWrapper.handleFiles);
     } else {
       this.log.debug('Mocha < 6 detected. Using custom logic to discover files');
       this.testFileNames = this.legacyDiscoverFiles();
     }
+    await this.additionalRequires();
   }
 
   private mocha6DiscoverFiles(handleFiles: (options: MochaOptions) => string[]): string[] {
@@ -92,7 +93,7 @@ export class MochaTestRunner implements TestRunner {
     return new Promise<RunResult>((resolve, reject) => {
       try {
         this.purgeFiles();
-        const mocha = new LibWrapper.Mocha({ reporter: StrykerMochaReporter as any, bail: true });
+        const mocha = new LibWrapper.Mocha({ reporter: StrykerMochaReporter as any, bail: true, rootHooks: this.rootHooks } as Mocha.MochaOptions);
         this.configure(mocha);
         this.addTestHooks(mocha, testHooks);
         this.addFiles(mocha);
@@ -162,10 +163,17 @@ export class MochaTestRunner implements TestRunner {
     }
   }
 
-  private additionalRequires() {
+  private async additionalRequires() {
     if (this.mochaOptions.require) {
-      const modulesToRequire = this.mochaOptions.require.map((moduleName) => (moduleName.startsWith('.') ? path.resolve(moduleName) : moduleName));
-      modulesToRequire.forEach(LibWrapper.require);
+      if (LibWrapper.handleRequires) {
+        const rawRootHooks = await LibWrapper.handleRequires(this.mochaOptions.require);
+        if (rawRootHooks) {
+          this.rootHooks = await LibWrapper.loadRootHooks!(rawRootHooks);
+        }
+      } else {
+        const modulesToRequire = this.mochaOptions.require.map((moduleName) => (moduleName.startsWith('.') ? path.resolve(moduleName) : moduleName));
+        modulesToRequire.forEach(LibWrapper.require);
+      }
     }
   }
 }

packages/mocha-runner/test/integration/MochaOptionsLoader.it.spec.ts

@@ -25,7 +25,7 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
     expect(actualConfig).deep.eq({
       ...DEFAULT_MOCHA_OPTIONS,
       config: configFile,
-      opts: false, // mocha sets opts: false after loading it...
+      opts: './test/mocha.opts',
       package: false, // mocha sets package: false after loading it...
       extension: ['js'],
       timeout: 2000,
@@ -38,7 +38,7 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
     const actualConfig = actLoad({ config: configFile });
     expect(actualConfig).deep.eq({
       ...DEFAULT_MOCHA_OPTIONS,
-      opts: false, // mocha sets opts: false after loading it...
+      opts: './test/mocha.opts',
       package: false, // mocha sets package: false after loading it...
       config: configFile,
       extension: ['json', 'js'],
@@ -53,7 +53,7 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
     expect(actualConfig).deep.eq({
       ...DEFAULT_MOCHA_OPTIONS,
       config: configFile,
-      opts: false, // mocha sets opts: false after loading it...
+      opts: './test/mocha.opts',
       package: false, // mocha sets package: false after loading it...
       extension: ['jsonc', 'js'],
       timeout: 2000,
@@ -68,7 +68,7 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
       ...DEFAULT_MOCHA_OPTIONS,
       'async-only': false,
       config: configFile,
-      opts: false, // mocha sets opts: false after loading it...
+      opts: './test/mocha.opts',
       package: false, // mocha sets package: false after loading it...
       extension: ['yml', 'js'],
       file: ['/path/to/some/file', '/path/to/some/other/file'],
@@ -80,24 +80,6 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
     });
   });
 
-  it('should support loading from "mocha.opts" (including providing files)', () => {
-    const configFile = resolveMochaConfig('mocha.opts');
-    const actualConfig = actLoad({ opts: configFile });
-    expect(actualConfig).deep.eq({
-      ...DEFAULT_MOCHA_OPTIONS,
-      'async-only': true,
-      extension: ['js'],
-      config: false, // mocha sets config: false after loading it...
-      package: false, // mocha sets package: false after loading it...
-      file: [],
-      ignore: [],
-      opts: configFile,
-      spec: ['/tests/**/*.js', '/foo/*.js'],
-      timeout: 2000,
-      ui: 'bdd',
-    });
-  });
-
   it('should support loading from "package.json"', () => {
     const pkgFile = resolveMochaConfig('package.json');
     const actualConfig = actLoad({ package: pkgFile });
@@ -128,7 +110,7 @@ describe(`${MochaOptionsLoader.name} integration`, () => {
       'no-opts': true,
     });
     const expectedOptions = {
-      extension: ['js'],
+      extension: ['js', 'cjs', 'mjs'],
       ['no-config']: true,
       ['no-opts']: true,
       ['no-package']: true,

packages/mocha-runner/test/integration/ProjectWithRootHooks.it.spec.ts

@@ -0,0 +1,34 @@
+import * as path from 'path';
+
+import { commonTokens } from '@stryker-mutator/api/plugin';
+import { testInjector } from '@stryker-mutator/test-helpers';
+import { expect } from 'chai';
+import { RunStatus } from '@stryker-mutator/api/test_runner';
+
+import { MochaTestRunner } from '../../src/MochaTestRunner';
+import MochaOptionsEditor from '../../src/MochaOptionsEditor';
+import MochaOptionsLoader from '../../src/MochaOptionsLoader';
+
+describe('Running a project with root hooks', () => {
+  const cwd = process.cwd();
+
+  let sut: MochaTestRunner;
+
+  beforeEach(async () => {
+    process.chdir(path.resolve(__dirname, '..', '..', 'testResources', 'parallel-with-root-hooks-sample'));
+    testInjector.injector.provideClass('loader', MochaOptionsLoader).injectClass(MochaOptionsEditor).edit(testInjector.options);
+    sut = testInjector.injector.provideValue(commonTokens.sandboxFileNames, []).injectClass(MochaTestRunner);
+    await sut.init();
+  });
+
+  afterEach(() => {
+    process.chdir(cwd);
+  });
+
+  it('should have run the root hooks', async () => {
+    const result = await sut.run({});
+    expect(result.status).eq(RunStatus.Complete);
+    expect(result.tests).has.lengthOf(2);
+    expect(result.errorMessages).lengthOf(0);
+  });
+});

packages/mocha-runner/test/unit/MochaOptionsLoader.spec.ts

@@ -106,6 +106,12 @@ describe(MochaOptionsLoader.name, () => {
       const actualOptions = sut.load(options);
       expect(actualOptions).deep.eq(createMochaOptions());
     });
+
+    it('should not allow to set parallel', () => {
+      rawOptions['parallel'] = true;
+      const actualOptions = sut.load(options);
+      expect((actualOptions as any).parallel).undefined;
+    });
   });
 
   describe('with mocha < 6', () => {

packages/mocha-runner/test/unit/MochaTestRunner.spec.ts

@@ -20,11 +20,15 @@ describe(MochaTestRunner.name, () => {
   let sut: MochaTestRunner;
   let requireStub: sinon.SinonStub;
   let handleFilesStub: sinon.SinonStub;
+  let handleRequiresStub: sinon.SinonStub;
+  let loadRootHooks: sinon.SinonStub;
 
   beforeEach(() => {
     MochaStub = sinon.stub(LibWrapper, 'Mocha');
     requireStub = sinon.stub(LibWrapper, 'require');
     handleFilesStub = sinon.stub(LibWrapper, 'handleFiles');
+    handleRequiresStub = sinon.stub(LibWrapper, 'handleRequires');
+    loadRootHooks = sinon.stub(LibWrapper, 'loadRootHooks');
     sinon.stub(utils, 'evalGlobal');
     mocha = sinon.createStubInstance(Mocha) as any;
     mocha.suite = sinon.createStubInstance(EventEmitter) as Mocha.Suite & sinon.SinonStubbedInstance<EventEmitter>;
@@ -108,7 +112,7 @@ describe(MochaTestRunner.name, () => {
       actAssertMatchedPatterns(undefined, expectedGlobPatterns);
     });
 
-    it('should throw an error if no files could be discovered', () => {
+    it('should reject if no files could be discovered', async () => {
       // Arrange
       multimatchStub.returns([]);
       const files = ['foo.js', 'bar.js'];
@@ -118,10 +122,10 @@ describe(MochaTestRunner.name, () => {
 
       // Act
       sut = createSut({ fileNames: files });
-      const actFn = () => sut.init();
+      const onGoingWork = sut.init();
 
       // Assert
-      expect(actFn).throws(
+      await expect(onGoingWork).rejectedWith(
         `[MochaTestRunner] No files discovered (tried pattern(s) ${relativeGlobbing}). Please specify the files (glob patterns) containing your tests in mochaOptions.spec in your config file.`
       );
       expect(testInjector.logger.debug).calledWith(`Tried ${absoluteGlobbing} on files: ${filesStringified}.`);
@@ -199,17 +203,21 @@ describe(MochaTestRunner.name, () => {
       expect(mocha.asyncOnly).not.called;
     });
 
-    it('should pass require additional require options when constructed', () => {
+    it('should pass require additional require options when constructed', async () => {
+      handleRequiresStub.value(undefined);
       const mochaOptions: Partial<MochaOptions> = { require: ['ts-node', 'babel-register'] };
-      createSut({ mochaOptions });
+      sut = createSut({ mochaOptions });
+      await sut.init();
       expect(requireStub).calledTwice;
       expect(requireStub).calledWith('ts-node');
       expect(requireStub).calledWith('babel-register');
     });
 
-    it('should pass and resolve relative require options when constructed', () => {
+    it('should pass and resolve relative require options when constructed', async () => {
+      handleRequiresStub.value(undefined);
       const mochaOptions: Partial<MochaOptions> = { require: ['./setup.js', 'babel-register'] };
-      createSut({ mochaOptions });
+      sut = createSut({ mochaOptions });
+      await sut.init();
       const resolvedRequire = path.resolve('./setup.js');
       expect(requireStub).calledTwice;
       expect(requireStub).calledWith(resolvedRequire);
@@ -246,6 +254,32 @@ describe(MochaTestRunner.name, () => {
     });
   });
 
+  describe('when mocha version >=8', () => {
+    beforeEach(() => {
+      handleFilesStub.returns(['src/math.js', 'test/math.spec.js']);
+    });
+
+    it('should handle require and allow for rootHooks', async () => {
+      handleRequiresStub.resolves(['root-hook1', 'bar-hook']);
+      const mochaOptions: Partial<MochaOptions> = { require: ['./setup.js', 'babel-register'] };
+      sut = createSut({ mochaOptions });
+      await sut.init();
+      expect(handleRequiresStub).calledWithExactly(['./setup.js', 'babel-register']);
+      expect(loadRootHooks).calledWithExactly(['root-hook1', 'bar-hook']);
+    });
+
+    it('should pass rootHooks to the mocha instance', async () => {
+      handleRequiresStub.resolves(['root-hook1', 'bar-hook']);
+      const rootHooks = { beforeEach() {} };
+      loadRootHooks.resolves(rootHooks);
+      const mochaOptions: Partial<MochaOptions> = { require: ['./setup.js', 'babel-register'] };
+      sut = createSut({ mochaOptions });
+      await sut.init();
+      await actRun();
+      expect(LibWrapper.Mocha).calledWithMatch({ rootHooks });
+    });
+  });
+
   async function actRun(options: RunOptions = { timeout: 0 }) {
     mocha.run.callsArg(0);
     return sut.run(options);

packages/mocha-runner/testResources/parallel-with-root-hooks-sample/.mocharc.jsonc

@@ -0,0 +1,6 @@
+{
+  "ui": "bdd",
+  "parallel": true,
+  "require": "./test/setup.js",
+  "spec": "test/unit/*.js"
+}

packages/mocha-runner/testResources/parallel-with-root-hooks-sample/test/setup.js

@@ -0,0 +1,5 @@
+exports.mochaHooks = {
+  beforeEach() {
+    global.add = (a, b) => a + b;
+  }
+}

packages/mocha-runner/testResources/parallel-with-root-hooks-sample/test/unit/spec.js

@@ -0,0 +1,8 @@
+const assert = require('assert');
+
+describe('add', () => {
+  it('should add 1 + 1 = 2', () => {
+    assert.equal(add(1, 1), 2);
+  });
+})
+

packages/mocha-runner/testResources/parallel-with-root-hooks-sample/test/unit/spec2.js

@@ -0,0 +1,7 @@
+const assert = require('assert');
+
+describe('add also', () => {
+  it('should add 2 - 3 = -1', () => {
+    assert.equal(add(2, -3), -1);
+  });
+})