diff --git a/packages/api/schema/stryker-core.json b/packages/api/schema/stryker-core.json index 6c7528a82f..e7fd692c82 100644 --- a/packages/api/schema/stryker-core.json +++ b/packages/api/schema/stryker-core.json @@ -398,6 +398,12 @@ "type": "number", "default": 5000 }, + "dryRunTimeoutMinutes": { + "description": "Configure an absolute timeout for the initial test run. (It can take a while, therefore we use minutes as time unit.)", + "type": "number", + "minimum": 0, + "default": 5 + }, "tsconfigFile": { "description": "Configure the (root) tsconfig file for typescript projects. This will allow Stryker to rewrite the `extends` and `references` settings in this and related tsconfig files in your sandbox. Defaults to `tsconfig.json`. This setting is also used when you enable the `@stryker-mutator/typescript-checker plugin", "type": "string", diff --git a/packages/core/README.md b/packages/core/README.md index bbbf2a0bc8..66b2bfac02 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -107,6 +107,7 @@ You can *ignore* files by adding an exclamation mark (`!`) at the start of an ex * [commandRunner](#commandRunner) * [coverageAnalysis](#coverageAnalysis) * [dashboard.*](#dashboard) +* [dryRunTimeoutMinutes](#dryRunTimeoutMinutes) * [fileLogLevel](#fileLogLevel) * [files](#files-string) * [logLevel](#logLevel) @@ -236,6 +237,15 @@ When using the config file you can provide an array with `string`s You can *ignore* files by adding an exclamation mark (`!`) at the start of an expression. + +### `dryRunTimeoutMinutes` [`number`] + +Default: `5` +Command line: `--dryRunTimeoutMinutes 5` +Config file: `dryRunTimeoutMinutes: 5` + +Use this option to configure an absolute timeout for the initial test run. Since it can take a while we use minutes as time unit. + ### `logLevel` [`string`] diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 77df1ef56a..c1a0cfc77c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,5 @@ import Stryker from './stryker'; import StrykerCli from './stryker-cli'; + export { Stryker, StrykerCli }; export default Stryker; diff --git a/packages/core/src/process/3-dry-run-executor.ts b/packages/core/src/process/3-dry-run-executor.ts index 1f1320e33c..7bcdbc338f 100644 --- a/packages/core/src/process/3-dry-run-executor.ts +++ b/packages/core/src/process/3-dry-run-executor.ts @@ -32,10 +32,6 @@ import { ConcurrencyTokenProvider } from '../concurrent'; import { MutationTestContext } from './4-mutation-test-executor'; import { MutantInstrumenterContext } from './2-mutant-instrumenter-executor'; -// The initial run might take a while. -// For example: angular-bootstrap takes up to 45 seconds. -// Lets take 5 minutes just to be sure -const INITIAL_RUN_TIMEOUT = 60 * 1000 * 5; const INITIAL_TEST_RUN_MARKER = 'Initial test run'; export interface DryRunContext extends MutantInstrumenterContext { @@ -70,6 +66,8 @@ function isFailedTest(testResult: TestResult): testResult is FailedTestResult { } export class DryRunExecutor { + private readonly dryRunTimeout: number; + public static readonly inject = tokens( commonTokens.injector, commonTokens.logger, @@ -125,9 +123,11 @@ export class DryRunExecutor { } private async timeDryRun(testRunner: TestRunner): Promise<{ dryRunResult: CompleteDryRunResult; timing: Timing }> { + const dryRunTimeout = this.options.dryRunTimeoutMinutes * 1000 * 60; this.timer.mark(INITIAL_TEST_RUN_MARKER); this.log.info('Starting initial test run. This may take a while.'); - const dryRunResult = await testRunner.dryRun({ timeout: INITIAL_RUN_TIMEOUT, coverageAnalysis: this.options.coverageAnalysis }); + this.log.debug(`Using timeout of ${this.dryRunTimeout} ms.`); + const dryRunResult = await testRunner.dryRun({ timeout: dryRunTimeout, coverageAnalysis: this.options.coverageAnalysis }); const grossTimeMS = this.timer.elapsedMs(INITIAL_TEST_RUN_MARKER); const humanReadableTimeElapsed = this.timer.humanReadableElapsed(INITIAL_TEST_RUN_MARKER); this.validateResultCompleted(dryRunResult); diff --git a/packages/core/src/stryker-cli.ts b/packages/core/src/stryker-cli.ts index 2101f70efd..0c564b7310 100644 --- a/packages/core/src/stryker-cli.ts +++ b/packages/core/src/stryker-cli.ts @@ -86,6 +86,7 @@ export default class StrykerCli { ) .option('--timeoutMS ', 'Tweak the absolute timeout used to wait for a test runner to complete', parseInt) .option('--timeoutFactor ', 'Tweak the standard deviation relative to the normal test run of a mutated test', parseFloat) + .option('--dryRunTimeoutMinutes ', 'Configure an absolute timeout for the initial test run. (It can take a while.)', parseFloat) .option('--maxConcurrentTestRunners ', 'Set the number of max concurrent test runner to spawn (default: cpuCount)', parseInt) .option( '-c, --concurrency ', diff --git a/packages/core/test/unit/config/options-validator.spec.ts b/packages/core/test/unit/config/options-validator.spec.ts index 790a34c995..00d9d0c7b4 100644 --- a/packages/core/test/unit/config/options-validator.spec.ts +++ b/packages/core/test/unit/config/options-validator.spec.ts @@ -61,6 +61,16 @@ describe(OptionsValidator.name, () => { actValidationErrors('Config option "timeoutFactor" has the wrong type. It should be a number, but was a string.'); }); + it('should be invalid with non-numeric dryRunTimeout', () => { + breakConfig('dryRunTimeoutMinutes', 'break'); + actValidationErrors('Config option "dryRunTimeoutMinutes" has the wrong type. It should be a number, but was a string.'); + }); + + it('should be invalid with negative numeric dryRunTimeout', () => { + breakConfig('dryRunTimeoutMinutes', -1); + actValidationErrors('Config option "dryRunTimeoutMinutes" should be >= 0, was -1.'); + }); + describe('plugins', () => { it('should be invalid with non-array plugins', () => { breakConfig('plugins', '@stryker-mutator/typescript'); diff --git a/packages/core/test/unit/process/3-dry-run-executor.spec.ts b/packages/core/test/unit/process/3-dry-run-executor.spec.ts index 774770ea59..7ed3be3a55 100644 --- a/packages/core/test/unit/process/3-dry-run-executor.spec.ts +++ b/packages/core/test/unit/process/3-dry-run-executor.spec.ts @@ -38,6 +38,33 @@ describe(DryRunExecutor.name, () => { await expect(sut.execute()).rejectedWith(expectedError); }); + describe('check timeout', () => { + let runResult: CompleteDryRunResult; + + beforeEach(() => { + runResult = factory.completeDryRunResult(); + testRunnerMock.dryRun.resolves(runResult); + runResult.tests.push(factory.successTestResult()); + }); + + it('should use the configured timeout in ms if option provided', async () => { + testInjector.options.dryRunTimeoutMinutes = 7.5; + const timeoutMS = testInjector.options.dryRunTimeoutMinutes * 60 * 1000; + await sut.execute(); + expect(testRunnerMock.dryRun).calledWithMatch({ + timeout: timeoutMS, + }); + }); + + it('should use the default timeout value if option not provided', async () => { + const defaultTimeoutMS = 5 * 60 * 1000; + await sut.execute(); + expect(testRunnerMock.dryRun).calledWithMatch({ + timeout: defaultTimeoutMS, + }); + }); + }); + describe('when the dryRun completes', () => { let runResult: CompleteDryRunResult; diff --git a/packages/core/test/unit/stryker-cli.spec.ts b/packages/core/test/unit/stryker-cli.spec.ts index af4bc63047..8ab59ca7c6 100644 --- a/packages/core/test/unit/stryker-cli.spec.ts +++ b/packages/core/test/unit/stryker-cli.spec.ts @@ -36,6 +36,7 @@ describe(StrykerCli.name, () => { [['--appendPlugins', 'foo,bar'], { appendPlugins: ['foo', 'bar'] }], [['--timeoutMS', '42'], { timeoutMS: 42 }], [['--timeoutFactor', '42'], { timeoutFactor: 42 }], + [['--dryRunTimeoutMinutes', '10'], { dryRunTimeoutMinutes: 10 }], [['--maxConcurrentTestRunners', '42'], { maxConcurrentTestRunners: 42 }], [['--tempDirName', 'foo-tmp'], { tempDirName: 'foo-tmp' }], [['--testRunner', 'foo-running'], { testRunner: 'foo-running' }],