diff --git a/lib/browser_collection.js b/lib/browser_collection.js index 0c624d762..d3fff4d8c 100644 --- a/lib/browser_collection.js +++ b/lib/browser_collection.js @@ -46,7 +46,24 @@ class BrowserCollection { return this.browsers.map((browser) => browser.serialize()) } - getResults () { + calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) { + if (results.disconnected || singleRunBrowserNotCaptured) { + return 1 + } + if (results.success + results.failed === 0 && !failOnEmptyTestSuite) { + return 0 + } + if (results.error) { + return 1 + } + if (failOnFailingTestSuite === false) { + // Tests executed without infrastructure error, exit with 0 independent of test status. + return 0 + } + return results.failed ? 1 : 0 + } + + getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) { const results = this.browsers.reduce((previous, current) => { previous.success += current.lastResult.success previous.failed += current.lastResult.failed @@ -56,7 +73,7 @@ class BrowserCollection { }, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0}) // compute exit status code - results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0 + results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) return results } diff --git a/lib/cli.js b/lib/cli.js index 1d71df41a..0d70273b8 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) { options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true' } + if (helper.isString(options.failOnFailingTestSuite)) { + options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true' + } + if (helper.isString(options.formatError)) { let required try { @@ -196,6 +200,8 @@ function describeStart () { .describe('report-slower-than', ' Report tests that are slower than given time [ms].') .describe('fail-on-empty-test-suite', 'Fail on empty test suite.') .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.') + .describe('fail-on-failing-test-suite', 'Fail on failing test suite.') + .describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.') .describe('help', 'Print usage and options.') } diff --git a/lib/server.js b/lib/server.js index 043c8a29c..045c4fee0 100644 --- a/lib/server.js +++ b/lib/server.js @@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter { const emitRunCompleteIfAllBrowsersDone = () => { if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) { - const results = singleRunBrowsers.getResults() - if (singleRunBrowserNotCaptured) { - results.exitCode = 1 - } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) { - results.exitCode = 0 - this.log.warn('Test suite was empty.') - } - this.emit('run_complete', singleRunBrowsers, results) + this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite)) } } diff --git a/test/unit/browser_collection.spec.js b/test/unit/browser_collection.spec.js index 417d2ad79..d883cb701 100644 --- a/test/unit/browser_collection.spec.js +++ b/test/unit/browser_collection.spec.js @@ -258,4 +258,90 @@ describe('BrowserCollection', () => { }) }) }) + + // ============================================================================ + // server.calculateExitCode + // ============================================================================ + describe('calculateExitCode', () => { + const EXIT_CODE_ERROR = 1 + const EXIT_CODE_SUCCESS = 0 + + describe('no tests', () => { + const results = { + success: 0, + failed: 0, + error: true + } + it('shall pass if failOnEmptyTestSuite not is set', () => { + const res = collection.calculateExitCode(results) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnEmptyTestSuite is false', () => { + const res = collection.calculateExitCode(results, false, false) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall fail if failOnEmptyTestSuite is true', () => { + const res = collection.calculateExitCode(results, false, true) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall fail if failOnFailingTestSuite is set', () => { + const res = collection.calculateExitCode(results, false, true, true) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + }) + describe('all test passed, no errors', () => { + const results = { + success: 10, + failed: 0, + error: false + } + it('shall fail if singleRunBrowserNotCaptured is true', () => { + const res = collection.calculateExitCode(results, true) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall pass if failOnEmptyTestSuite not is set', () => { + const res = collection.calculateExitCode(results, false) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnEmptyTestSuite not is set', () => { + const res = collection.calculateExitCode(results, false, false) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnFailingTestSuite is true', () => { + const res = collection.calculateExitCode(results, false, true, true) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + it('shall pass if failOnFailingTestSuite is false', () => { + const res = collection.calculateExitCode(results, false, true, false) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + }) + describe('all test passed, with error', () => { + const results = { + success: 10, + failed: 5, + error: false + } + it('shall fail if singleRunBrowserNotCaptured is true', () => { + const res = collection.calculateExitCode(results, true) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall fail if failOnEmptyTestSuite not is set', () => { + const res = collection.calculateExitCode(results, false) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall fail if failOnEmptyTestSuite not is set', () => { + const res = collection.calculateExitCode(results, false, false) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall fail if failOnFailingTestSuite is true', () => { + const res = collection.calculateExitCode(results, false, true, true) + expect(res).to.be.equal(EXIT_CODE_ERROR) + }) + it('shall pass if failOnFailingTestSuite is false', () => { + const res = collection.calculateExitCode(results, false, true, false) + expect(res).to.be.equal(EXIT_CODE_SUCCESS) + }) + }) + }) })