feat(debugging): allow passing node args to the test runner (#2609)

Add the `--testRunnerNodeArgs` option. With this option, you can pass arguments to the node process that hosts the test runner.

For example, running stryker with `--timeoutMS 9999999 --concurrency 1 --testRunnerNodeArgs --inspect-brk` will allow you to debug the tests running in the dryRun or the mutation test run.

Another use case is using `--testRunnerNodeArgs --cpu-prof`, this allows you to analyze the performance of the child process.

Using "testRunnerNodeArgs" together with the "command" test runner is not supported. Doing so will result in a warning.

packages/api/schema/stryker-core.json

@@ -400,6 +400,14 @@
       "type": "string",
       "default": "command"
     },
+    "testRunnerNodeArgs": {
+      "description": "Configure arguments to be passed as exec arguments to the test runner child process. For example, running Stryker with `--timeoutMS 9999999 --concurrency 1 --testRunnerNodeArgs --inspect-brk` will allow you to debug the test runner child process. See `execArgv` of [`child_process.fork`](https://nodejs.org/api/child_process.html#child_process_child_process_fork_modulepath_args_options)",
+      "type": "array",
+      "default": [],
+      "items": {
+        "type": "string"
+      }
+    },
     "thresholds": {
       "description": "Specify the thresholds for mutation score.",
       "$ref": "#/definitions/mutationScoreThresholds",

packages/core/README.md

@@ -395,9 +395,9 @@ _Note: Use of "testFramework" is no longer needed. You can remove it from your c
 <a name="testRunner"></a>
 ### `testRunner` [`string`]
 
-Default: `'command'`
-Command line: `--testRunner karma`
-Config file: `testRunner: 'karma'`
+Default: `'command'`  
+Command line: `--testRunner karma`  
+Config file: `testRunner: 'karma'`  
 
 With `testRunner` you specify the test runner that Stryker uses to run your tests. The default value is `command`. The command runner runs a configurable bash/cmd command and bases the result on the exit code of that program (0 for success, otherwise failed). You can configure this command via the config file using the `commandRunner: { command: 'npm run mocha' }`. It uses `npm test` as the command by default.
 
@@ -407,6 +407,15 @@ If possible, you should try to use one of the test runner plugins that hook into
 For example: install and use the `stryker-karma-runner` to use `karma` as a test runner.
 See the [list of plugins](https://stryker-mutator.io/plugins.html) for an up-to-date list of supported test runners and plugins.
 
+<a name="testRunnerNodeArgs"></a>
+### `testRunnerNodeArgs` [`string[]`]
+
+Default: `[]`  
+Command line: `--testRunnerNodeArgs "--inspect-brk --cpu-prof"`  
+Config file: `testRunnerNodeArgs: ['--inspect-brk', '--cpu-prof']`  
+
+Configure arguments to be passed as exec arguments to the test runner child process. For example, running Stryker with `--timeoutMS 9999999 --concurrency 1 --testRunnerNodeArgs "--inspect-brk"` will allow you to debug the test runner child process. See `execArgv` of [`child_process.fork`](https://nodejs.org/api/child_process.html#child_process_child_process_fork_modulepath_args_options).
+
 <a name="thresholds"></a>
 ### `thresholds` [`object`]
 

packages/core/src/checker/checker-facade.ts

@@ -22,7 +22,15 @@ export class CheckerFacade implements Checker, Disposable, Worker {
 
   constructor(options: StrykerOptions, loggingContext: LoggingClientContext) {
     if (options.checkers.length) {
-      this.childProcess = ChildProcessProxy.create(require.resolve('./checker-worker'), loggingContext, options, {}, process.cwd(), CheckerWorker);
+      this.childProcess = ChildProcessProxy.create(
+        require.resolve('./checker-worker'),
+        loggingContext,
+        options,
+        {},
+        process.cwd(),
+        CheckerWorker,
+        []
+      );
     }
   }
 

packages/core/src/child-proxy/child-process-proxy.ts

@@ -46,11 +46,12 @@ export default class ChildProcessProxy<T> implements Disposable {
     loggingContext: LoggingClientContext,
     options: StrykerOptions,
     additionalInjectableValues: unknown,
-    workingDirectory: string
+    workingDirectory: string,
+    execArgv: string[]
   ) {
-    this.worker = fork(require.resolve('./child-process-proxy-worker'), [autoStart], { silent: true, execArgv: [] });
+    this.worker = fork(require.resolve('./child-process-proxy-worker'), [autoStart], { silent: true, execArgv });
     this.initTask = new Task();
-    this.log.debug('Starting %s in child process %s', requirePath, this.worker.pid);
+    this.log.debug('Started %s in child process %s%s', requireName, this.worker.pid, execArgv.length ? ` (using args ${execArgv.join(' ')})` : '');
     this.send({
       additionalInjectableValues,
       kind: WorkerMessageKind.Init,
@@ -77,9 +78,10 @@ export default class ChildProcessProxy<T> implements Disposable {
     options: StrykerOptions,
     additionalInjectableValues: TAdditionalContext,
     workingDirectory: string,
-    injectableClass: InjectableClass<TAdditionalContext & PluginContext, R, Tokens>
+    injectableClass: InjectableClass<TAdditionalContext & PluginContext, R, Tokens>,
+    execArgv: string[]
   ): ChildProcessProxy<R> {
-    return new ChildProcessProxy(requirePath, injectableClass.name, loggingContext, options, additionalInjectableValues, workingDirectory);
+    return new ChildProcessProxy(requirePath, injectableClass.name, loggingContext, options, additionalInjectableValues, workingDirectory, execArgv);
   }
 
   private send(message: WorkerMessage) {
@@ -170,6 +172,10 @@ export default class ChildProcessProxy<T> implements Disposable {
     return this.stdoutBuilder.toString();
   }
 
+  public get stderr() {
+    return this.stderrBuilder.toString();
+  }
+
   private reportError(error: Error) {
     this.workerTasks.filter((task) => !task.isCompleted).forEach((task) => task.reject(error));
   }

packages/core/src/config/options-validator.ts

@@ -11,6 +11,8 @@ import { coreTokens } from '../di';
 import { ConfigError } from '../errors';
 import { isWarningEnabled } from '../utils/object-utils';
 
+import CommandTestRunner from '../test-runner/command-test-runner';
+
 import { describeErrors } from './validation-errors';
 
 const ajv = new Ajv({ useDefaults: true, allErrors: true, jsonPointers: false, verbose: true, missingRefs: 'ignore', logger: false });
@@ -79,6 +81,11 @@ export class OptionsValidator {
         options.concurrency = options.maxConcurrentTestRunners;
       }
     }
+    if (CommandTestRunner.is(options.testRunner) && options.testRunnerNodeArgs.length) {
+      this.log.warn(
+        'Using "testRunnerNodeArgs" together with the "command" test runner is not supported, these arguments will be ignored. You can add your custom arguments by setting the "commandRunner.command" option.'
+      );
+    }
     additionalErrors.forEach((error) => this.log.error(error));
     this.throwErrorIfNeeded(additionalErrors);
   }

packages/core/src/stryker-cli.ts

@@ -20,8 +20,10 @@ function deepOption<T extends string, R>(object: { [K in T]?: R }, key: T) {
   };
 }
 
-function list(val: string) {
-  return val.split(',');
+const list = createSplitter(',');
+
+function createSplitter(sep: string) {
+  return (val: string) => val.split(sep);
 }
 
 function parseBoolean(val: string) {
@@ -77,6 +79,11 @@ export default class StrykerCli {
         `The coverage analysis strategy you want to use. Default value: "${defaultValues.coverageAnalysis}"`
       )
       .option('--testRunner <name>', 'The name of the test runner you want to use')
+      .option(
+        '--testRunnerNodeArgs <listOfNodeArgs>',
+        'A comma separated list of node args to be passed to test runner child processes.',
+        createSplitter(' ')
+      )
       .option('--reporters <name>', 'A comma separated list of the names of the reporter(s) you want to use', list)
       .option('--plugins <listOfPlugins>', 'A list of plugins you want stryker to load (`require`).', list)
       .option(

packages/core/src/test-runner/child-process-test-runner-decorator.ts

@@ -23,7 +23,8 @@ export default class ChildProcessTestRunnerDecorator implements TestRunner {
       options,
       {},
       sandboxWorkingDirectory,
-      ChildProcessTestRunnerWorker
+      ChildProcessTestRunnerWorker,
+      options.testRunnerNodeArgs
     );
   }
 

packages/core/test/integration/child-proxy/child-process-proxy.it.spec.ts

@@ -30,7 +30,9 @@ describe(ChildProcessProxy.name, () => {
     const port = await loggingServer.listen();
     const options = testInjector.injector.resolve(commonTokens.options);
     log = currentLogMock();
-    sut = ChildProcessProxy.create(require.resolve('./echo'), { port, level: LogLevel.Debug }, options, { name: echoName }, workingDir, Echo);
+    sut = ChildProcessProxy.create(require.resolve('./echo'), { port, level: LogLevel.Debug }, options, { name: echoName }, workingDir, Echo, [
+      '--no-warnings', // test if node args are forwarded with this setting, see https://nodejs.org/api/cli.html#cli_no_warnings
+    ]);
   });
 
   afterEach(async () => {
@@ -68,6 +70,11 @@ describe(ChildProcessProxy.name, () => {
     expect(actual.name).eq('foobar.txt');
   });
 
+  it('should use `execArgv` to start the child process', async () => {
+    await sut.proxy.warning();
+    expect(sut.stderr).not.includes('Foo warning');
+  });
+
   it('should be able to receive a promise rejection', async () => {
     await expect(sut.proxy.reject('Foobar error')).rejectedWith('Foobar error');
   });

packages/core/test/integration/child-proxy/echo.ts

@@ -35,6 +35,10 @@ export class Echo {
     return new File('foobar.txt', 'hello foobar');
   }
 
+  public warning() {
+    process.emitWarning('Foo warning');
+  }
+
   public cwd() {
     return process.cwd();
   }

packages/core/test/unit/child-proxy/child-process-proxy.spec.ts

@@ -94,6 +94,17 @@ describe(ChildProcessProxy.name, () => {
       expect(childProcessMock.send).calledWith(serialize(expectedMessage));
     });
 
+    it('should log the exec arguments and require name', () => {
+      // Act
+      createSut({
+        loggingContext: LOGGING_CONTEXT,
+        execArgv: ['--cpu-prof', '--inspect'],
+      });
+
+      // Assert
+      expect(logMock.debug).calledWith('Started %s in child process %s%s', 'HelloClass', childProcessMock.pid, ' (using args --cpu-prof --inspect)');
+    });
+
     it('should listen to worker process', () => {
       createSut();
       expect(childProcessMock.listeners('message')).lengthOf(1);
@@ -103,6 +114,11 @@ describe(ChildProcessProxy.name, () => {
       createSut();
       expect(childProcessMock.listeners('close')).lengthOf(1);
     });
+
+    it('should set `execArgv`', () => {
+      createSut({ execArgv: ['--inspect-brk'] });
+      expect(forkStub).calledWithMatch(sinon.match.string, sinon.match.array, sinon.match({ execArgv: ['--inspect-brk'] }));
+    });
   });
 
   describe('on close', () => {
@@ -247,6 +263,7 @@ function createSut(
     options?: Partial<StrykerOptions>;
     workingDir?: string;
     name?: string;
+    execArgv?: string[];
   } = {}
 ): ChildProcessProxy<HelloClass> {
   return ChildProcessProxy.create(
@@ -255,6 +272,7 @@ function createSut(
     factory.strykerOptions(overrides.options),
     { name: overrides.name || 'someArg' },
     overrides.workingDir || 'workingDir',
-    HelloClass
+    HelloClass,
+    overrides.execArgv ?? []
   );
 }

packages/core/test/unit/config/options-validator.spec.ts

@@ -33,7 +33,8 @@ describe(OptionsValidator.name, () => {
     });
 
     it('should be invalid with thresholds.high null', () => {
-      (testInjector.options.thresholds.high as any) = null;
+      // @ts-expect-error invalid setting
+      testInjector.options.thresholds.high = null;
       actValidationErrors('Config option "thresholds.high" has the wrong type. It should be a number, but was a null.');
     });
 
@@ -45,7 +46,8 @@ describe(OptionsValidator.name, () => {
   });
 
   it('should be invalid with invalid logLevel', () => {
-    testInjector.options.logLevel = 'thisTestPasses' as any;
+    // @ts-expect-error invalid setting
+    testInjector.options.logLevel = 'thisTestPasses';
     actValidationErrors(
       'Config option "logLevel" should be one of the allowed values ("off", "fatal", "error", "warn", "info", "debug", "trace"), but was "thisTestPasses".'
     );
@@ -97,12 +99,14 @@ describe(OptionsValidator.name, () => {
 
   describe('mutator', () => {
     it('should be invalid with non-string mutator', () => {
-      (testInjector.options.mutator as any) = 1;
+      // @ts-expect-error invalid setting
+      testInjector.options.mutator = 1;
       actValidationErrors('Config option "mutator" has the wrong type. It should be a object, but was a number.');
     });
 
     it('should report a deprecation warning for "mutator.name"', () => {
-      (testInjector.options.mutator as any) = {
+      testInjector.options.mutator = {
+        // @ts-expect-error invalid setting
         name: 'javascript',
       };
       sut.validate(testInjector.options);
@@ -112,7 +116,8 @@ describe(OptionsValidator.name, () => {
     });
 
     it('should report a deprecation warning for mutator as a string', () => {
-      (testInjector.options.mutator as any) = 'javascript';
+      // @ts-expect-error invalid setting
+      testInjector.options.mutator = 'javascript';
       sut.validate(testInjector.options);
       expect(testInjector.logger.warn).calledWith(
         'DEPRECATED. Use of "mutator" as string is no longer needed. You can remove it from your configuration. Stryker now supports mutating of JavaScript and friend files out of the box.'
@@ -122,7 +127,7 @@ describe(OptionsValidator.name, () => {
 
   describe('testFramework', () => {
     it('should report a deprecation warning', () => {
-      (testInjector.options as any).testFramework = '';
+      testInjector.options.testFramework = '';
       sut.validate(testInjector.options);
       expect(testInjector.logger.warn).calledWith(
         'DEPRECATED. Use of "testFramework" is no longer needed. You can remove it from your configuration. Your test runner plugin now handles its own test framework integration.'
@@ -192,9 +197,18 @@ describe(OptionsValidator.name, () => {
     actValidationErrors('Config option "maxTestRunnerReuse" has the wrong type. It should be a number, but was a string.');
   });
 
+  it('should warn when testRunnerNodeArgs are combined with the "command" test runner', () => {
+    testInjector.options.testRunnerNodeArgs = ['--inspect-brk'];
+    testInjector.options.testRunner = 'command';
+    sut.validate(testInjector.options);
+    expect(testInjector.logger.warn).calledWith(
+      'Using "testRunnerNodeArgs" together with the "command" test runner is not supported, these arguments will be ignored. You can add your custom arguments by setting the "commandRunner.command" option.'
+    );
+  });
+
   describe('transpilers', () => {
     it('should report a deprecation warning', () => {
-      (testInjector.options.transpilers as any) = ['stryker-jest'];
+      testInjector.options.transpilers = ['stryker-jest'];
       sut.validate(testInjector.options);
       expect(testInjector.logger.warn).calledWith(
         'DEPRECATED. Support for "transpilers" is removed. You can now configure your own "buildCommand". For example, npm run build.'

packages/core/test/unit/stryker-cli.spec.ts

@@ -40,6 +40,7 @@ describe(StrykerCli.name, () => {
       [['--maxConcurrentTestRunners', '42'], { maxConcurrentTestRunners: 42 }],
       [['--tempDirName', 'foo-tmp'], { tempDirName: 'foo-tmp' }],
       [['--testRunner', 'foo-running'], { testRunner: 'foo-running' }],
+      [['--testRunnerNodeArgs', '--inspect=1337 --gc'], { testRunnerNodeArgs: ['--inspect=1337', '--gc'] }],
       [['--coverageAnalysis', 'all'], { coverageAnalysis: 'all' }],
       [['--concurrency', '5'], { concurrency: 5 }],
       [['--cleanTempDir', 'false'], { cleanTempDir: false }],

packages/core/test/unit/test-runner/child-process-test-runner-decorator.spec.ts

@@ -15,7 +15,6 @@ import ChildProcessTestRunnerDecorator from '../../../src/test-runner/child-proc
 import { ChildProcessTestRunnerWorker } from '../../../src/test-runner/child-process-test-runner-worker';
 
 describe(ChildProcessTestRunnerDecorator.name, () => {
-  let sut: ChildProcessTestRunnerDecorator;
   let options: StrykerOptions;
   let childProcessProxyMock: {
     proxy: sinon.SinonStubbedInstance<Required<TestRunner>>;
@@ -37,26 +36,34 @@ describe(ChildProcessTestRunnerDecorator.name, () => {
       plugins: ['foo-plugin', 'bar-plugin'],
     });
     loggingContext = { port: 4200, level: LogLevel.Fatal };
-    sut = new ChildProcessTestRunnerDecorator(options, 'a working directory', loggingContext);
   });
 
+  function createSut(): ChildProcessTestRunnerDecorator {
+    return new ChildProcessTestRunnerDecorator(options, 'a working directory', loggingContext);
+  }
+
   it('should create the child process proxy', () => {
-    expect(childProcessProxyCreateStub).calledWith(
+    options.testRunnerNodeArgs = ['--inspect', '--no-warnings'];
+    createSut();
+    expect(childProcessProxyCreateStub).calledWithExactly(
       require.resolve('../../../src/test-runner/child-process-test-runner-worker.js'),
       loggingContext,
       options,
       {},
       'a working directory',
-      ChildProcessTestRunnerWorker
+      ChildProcessTestRunnerWorker,
+      ['--inspect', '--no-warnings']
     );
   });
 
   it('should forward `init` calls', () => {
+    const sut = createSut();
     childProcessProxyMock.proxy.init.resolves(42);
     return expect(sut.init()).eventually.eq(42);
   });
 
   it('should forward `dryRun` calls', async () => {
+    const sut = createSut();
     const expectedResult = factory.completeDryRunResult({ mutantCoverage: factory.mutantCoverage() });
     childProcessProxyMock.proxy.dryRun.resolves(expectedResult);
     const runOptions = factory.dryRunOptions({
@@ -68,6 +75,7 @@ describe(ChildProcessTestRunnerDecorator.name, () => {
   });
 
   it('should forward `mutantRun` calls', async () => {
+    const sut = createSut();
     const expectedResult = factory.survivedMutantRunResult();
     childProcessProxyMock.proxy.mutantRun.resolves(expectedResult);
     const runOptions = factory.mutantRunOptions({
@@ -80,18 +88,21 @@ describe(ChildProcessTestRunnerDecorator.name, () => {
 
   describe('dispose', () => {
     it('should dispose the test runner before disposing the child process itself on `dispose`', async () => {
+      const sut = createSut();
       childProcessProxyMock.proxy.dispose.resolves();
       await sut.dispose();
       expect(childProcessProxyMock.proxy.dispose).calledBefore(childProcessProxyMock.dispose);
     });
 
     it('should not reject when the child process is down', async () => {
+      const sut = createSut();
       childProcessProxyMock.proxy.dispose.rejects(new ChildProcessCrashedError(1, '1'));
       await sut.dispose();
       expect(childProcessProxyMock.dispose).called;
     });
 
     it('should only wait 2 seconds for the test runner to be disposed', async () => {
+      const sut = createSut();
       const testRunnerDisposeTask = new Task();
       childProcessProxyMock.proxy.dispose.returns(testRunnerDisposeTask.promise);
       const disposePromise = sut.dispose();