feat(timeout): add --dryRunTimeoutMinutes option

Add the `--dryRunTimeoutMinutes` option. With it, you can override the default of 5 mins for the timeout of the dry run (the initial test run). Note: if running your tests takes more than 5 minutes, your mutation testing process might take a significant amount of time.

packages/api/schema/stryker-core.json

@@ -398,6 +398,12 @@
       "type": "number",
       "default": 5000
     },
+    "dryRunTimeoutMinutes": {
+      "description": "Configure an absolute timeout for the initial test run. (It can take a while, therefore we use minutes as time unit.)",
+      "type": "number",
+      "minimum": 0,
+      "default": 5
+    },
     "tsconfigFile": {
       "description": "Configure the (root) tsconfig file for typescript projects. This will allow Stryker to rewrite the `extends` and `references` settings in this and related tsconfig files in your sandbox. Defaults to `tsconfig.json`. This setting is also used when you enable the `@stryker-mutator/typescript-checker plugin",
       "type": "string",

packages/core/README.md

@@ -107,6 +107,7 @@ You can *ignore* files by adding an exclamation mark (`!`) at the start of an ex
 * [commandRunner](#commandRunner)
 * [coverageAnalysis](#coverageAnalysis)
 * [dashboard.*](#dashboard)
+* [dryRunTimeoutMinutes](#dryRunTimeoutMinutes)
 * [fileLogLevel](#fileLogLevel)
 * [files](#files-string)
 * [logLevel](#logLevel)
@@ -236,6 +237,15 @@ When using the config file you can provide an array with `string`s
 
 You can *ignore* files by adding an exclamation mark (`!`) at the start of an expression.
 
+<a name="dryRunTimeoutMinutes"></a>
+### `dryRunTimeoutMinutes` [`number`]
+
+Default: `5`  
+Command line: `--dryRunTimeoutMinutes 5`  
+Config file: `dryRunTimeoutMinutes: 5`  
+
+Use this option to configure an absolute timeout for the initial test run. Since it can take a while we use minutes as time unit.
+
 <a name="logLevel"></a>
 ### `logLevel` [`string`]
 

packages/core/src/index.ts

@@ -1,4 +1,5 @@
 import Stryker from './stryker';
 import StrykerCli from './stryker-cli';
+
 export { Stryker, StrykerCli };
 export default Stryker;

packages/core/src/process/3-dry-run-executor.ts

@@ -32,10 +32,6 @@ import { ConcurrencyTokenProvider } from '../concurrent';
 import { MutationTestContext } from './4-mutation-test-executor';
 import { MutantInstrumenterContext } from './2-mutant-instrumenter-executor';
 
-// The initial run might take a while.
-// For example: angular-bootstrap takes up to 45 seconds.
-// Lets take 5 minutes just to be sure
-const INITIAL_RUN_TIMEOUT = 60 * 1000 * 5;
 const INITIAL_TEST_RUN_MARKER = 'Initial test run';
 
 export interface DryRunContext extends MutantInstrumenterContext {
@@ -70,6 +66,8 @@ function isFailedTest(testResult: TestResult): testResult is FailedTestResult {
 }
 
 export class DryRunExecutor {
+  private readonly dryRunTimeout: number;
+
   public static readonly inject = tokens(
     commonTokens.injector,
     commonTokens.logger,
@@ -125,9 +123,11 @@ export class DryRunExecutor {
   }
 
   private async timeDryRun(testRunner: TestRunner): Promise<{ dryRunResult: CompleteDryRunResult; timing: Timing }> {
+    const dryRunTimeout = this.options.dryRunTimeoutMinutes * 1000 * 60;
     this.timer.mark(INITIAL_TEST_RUN_MARKER);
     this.log.info('Starting initial test run. This may take a while.');
-    const dryRunResult = await testRunner.dryRun({ timeout: INITIAL_RUN_TIMEOUT, coverageAnalysis: this.options.coverageAnalysis });
+    this.log.debug(`Using timeout of ${this.dryRunTimeout} ms.`);
+    const dryRunResult = await testRunner.dryRun({ timeout: dryRunTimeout, coverageAnalysis: this.options.coverageAnalysis });
     const grossTimeMS = this.timer.elapsedMs(INITIAL_TEST_RUN_MARKER);
     const humanReadableTimeElapsed = this.timer.humanReadableElapsed(INITIAL_TEST_RUN_MARKER);
     this.validateResultCompleted(dryRunResult);

packages/core/src/stryker-cli.ts

@@ -86,6 +86,7 @@ export default class StrykerCli {
       )
       .option('--timeoutMS <number>', 'Tweak the absolute timeout used to wait for a test runner to complete', parseInt)
       .option('--timeoutFactor <number>', 'Tweak the standard deviation relative to the normal test run of a mutated test', parseFloat)
+      .option('--dryRunTimeoutMinutes <number>', 'Configure an absolute timeout for the initial test run. (It can take a while.)', parseFloat)
       .option('--maxConcurrentTestRunners <n>', 'Set the number of max concurrent test runner to spawn (default: cpuCount)', parseInt)
       .option(
         '-c, --concurrency <n>',

packages/core/test/unit/config/options-validator.spec.ts

@@ -61,6 +61,16 @@ describe(OptionsValidator.name, () => {
     actValidationErrors('Config option "timeoutFactor" has the wrong type. It should be a number, but was a string.');
   });
 
+  it('should be invalid with non-numeric dryRunTimeout', () => {
+    breakConfig('dryRunTimeoutMinutes', 'break');
+    actValidationErrors('Config option "dryRunTimeoutMinutes" has the wrong type. It should be a number, but was a string.');
+  });
+
+  it('should be invalid with negative numeric dryRunTimeout', () => {
+    breakConfig('dryRunTimeoutMinutes', -1);
+    actValidationErrors('Config option "dryRunTimeoutMinutes" should be >= 0, was -1.');
+  });
+
   describe('plugins', () => {
     it('should be invalid with non-array plugins', () => {
       breakConfig('plugins', '@stryker-mutator/typescript');

packages/core/test/unit/process/3-dry-run-executor.spec.ts

@@ -38,6 +38,33 @@ describe(DryRunExecutor.name, () => {
     await expect(sut.execute()).rejectedWith(expectedError);
   });
 
+  describe('check timeout', () => {
+    let runResult: CompleteDryRunResult;
+
+    beforeEach(() => {
+      runResult = factory.completeDryRunResult();
+      testRunnerMock.dryRun.resolves(runResult);
+      runResult.tests.push(factory.successTestResult());
+    });
+
+    it('should use the configured timeout in ms if option provided', async () => {
+      testInjector.options.dryRunTimeoutMinutes = 7.5;
+      const timeoutMS = testInjector.options.dryRunTimeoutMinutes * 60 * 1000;
+      await sut.execute();
+      expect(testRunnerMock.dryRun).calledWithMatch({
+        timeout: timeoutMS,
+      });
+    });
+
+    it('should use the default timeout value if option not provided', async () => {
+      const defaultTimeoutMS = 5 * 60 * 1000;
+      await sut.execute();
+      expect(testRunnerMock.dryRun).calledWithMatch({
+        timeout: defaultTimeoutMS,
+      });
+    });
+  });
+
   describe('when the dryRun completes', () => {
     let runResult: CompleteDryRunResult;
 

packages/core/test/unit/stryker-cli.spec.ts

@@ -36,6 +36,7 @@ describe(StrykerCli.name, () => {
       [['--appendPlugins', 'foo,bar'], { appendPlugins: ['foo', 'bar'] }],
       [['--timeoutMS', '42'], { timeoutMS: 42 }],
       [['--timeoutFactor', '42'], { timeoutFactor: 42 }],
+      [['--dryRunTimeoutMinutes', '10'], { dryRunTimeoutMinutes: 10 }],
       [['--maxConcurrentTestRunners', '42'], { maxConcurrentTestRunners: 42 }],
       [['--tempDirName', 'foo-tmp'], { tempDirName: 'foo-tmp' }],
       [['--testRunner', 'foo-running'], { testRunner: 'foo-running' }],