Skip to content

Commit

Permalink
feat: Do not fail on empty test suite
Browse files Browse the repository at this point in the history
Add option to configuration-file and cli-args to server (single run) and runner,
for disabling failure on empty test suite. Will instead display a warning.

Closes #926
  • Loading branch information
Christian Budde Christensen committed Jan 11, 2016
1 parent 7f10d81 commit 8004763
Show file tree
Hide file tree
Showing 7 changed files with 95 additions and 14 deletions.
8 changes: 8 additions & 0 deletions lib/cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ var processArgs = function (argv, options, fs, path) {
options.colors = options.colors === 'true'
}

if (helper.isString(options.failOnEmptyTestSuite)) {
options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
}

if (helper.isString(options.logLevel)) {
var logConstant = constant['LOG_' + options.logLevel.toUpperCase()]
if (helper.isDefined(logConstant)) {
Expand Down Expand Up @@ -165,6 +169,8 @@ var describeStart = function () {
.describe('single-run', 'Run the test when browsers captured and exit.')
.describe('no-single-run', 'Disable single-run.')
.describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
.describe('help', 'Print usage and options.')
}

Expand All @@ -176,6 +182,8 @@ var describeRun = function () {
' $0 run [<configFile>] [<options>] [ -- <clientArgs>]')
.describe('port', '<integer> Port where the server is listening.')
.describe('no-refresh', 'Do not re-glob all the patterns.')
.describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
.describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
.describe('help', 'Print usage.')
}

Expand Down
1 change: 1 addition & 0 deletions lib/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ var Config = function () {
this.browserDisconnectTolerance = 0
this.browserNoActivityTimeout = 10000
this.concurrency = Infinity
this.failOnEmptyTestSuite = true
}

var CONFIG_SYNTAX_HELP = ' module.exports = function(config) {\n' +
Expand Down
3 changes: 2 additions & 1 deletion lib/middleware/runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ var createRunnerMiddleware = function (emitter, fileList, capturedBrowsers, repo
// clean up, close runner response
emitter.once('run_complete', function (browsers, results) {
reporter.removeAdapter(responseWrite)
response.end(constant.EXIT_CODE + results.exitCode)
var emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1
response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode)
})
})

Expand Down
20 changes: 14 additions & 6 deletions lib/runner.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ var http = require('http')
var constant = require('./constants')
var helper = require('./helper')
var cfg = require('./config')
var logger = require('./logger')
var log = logger.create('runner')

var parseExitCode = function (buffer, defaultCode) {
var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 1
var parseExitCode = function (buffer, defaultCode, failOnEmptyTestSuite) {
var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 2

if (tailPos < 0) {
return defaultCode
Expand All @@ -14,9 +16,15 @@ var parseExitCode = function (buffer, defaultCode) {
// tail buffer which might contain the message
var tail = buffer.slice(tailPos)
var tailStr = tail.toString()
if (tailStr.substr(0, tailStr.length - 1) === constant.EXIT_CODE) {
if (tailStr.substr(0, tailStr.length - 2) === constant.EXIT_CODE) {
tail.fill('\x00')
return parseInt(tailStr.substr(-1), 10)
var emptyInt = parseInt(tailStr.substr(-2, 1), 10)
var exitCode = parseInt(tailStr.substr(-1), 10)
if (failOnEmptyTestSuite === false && emptyInt === 0) {
log.warn('Test suite was empty.')
return 0
}
return exitCode
}

return defaultCode
Expand All @@ -40,7 +48,7 @@ exports.run = function (config, done) {

var request = http.request(options, function (response) {
response.on('data', function (buffer) {
exitCode = parseExitCode(buffer, exitCode)
exitCode = parseExitCode(buffer, exitCode, config.failOnEmptyTestSuite)
process.stdout.write(buffer)
})

Expand All @@ -51,7 +59,7 @@ exports.run = function (config, done) {

request.on('error', function (e) {
if (e.code === 'ECONNREFUSED') {
console.error('There is no server listening on port %d', options.port)
log.error('There is no server listening on port %d', options.port)
done(1, e.code)
} else {
throw e
Expand Down
4 changes: 3 additions & 1 deletion lib/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -251,8 +251,10 @@ Server.prototype._start = function (config, launcher, preprocess, fileList, webS
var results = singleRunBrowsers.getResults()
if (singleRunBrowserNotCaptured) {
results.exitCode = 1
} else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
results.exitCode = 0
self.log.warn('Test suite was empty.')
}

self.emit('run_complete', singleRunBrowsers, results)
}
}
Expand Down
50 changes: 49 additions & 1 deletion test/unit/middleware/runner.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ describe('middleware.runner', () => {

response.once('end', () => {
expect(nextSpy).to.not.have.been.called
expect(response).to.beServedAs(200, 'result\x1FEXIT0')
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
done()
})

Expand All @@ -83,6 +83,54 @@ describe('middleware.runner', () => {
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0})
})

it('should set the empty to 0 if empty results', (done) => {
capturedBrowsers.add(new Browser())
sinon.stub(capturedBrowsers, 'areAllReady', () => true)

response.once('end', () => {
expect(nextSpy).to.not.have.been.called
expect(response).to.beServedAs(200, 'result\x1FEXIT00')
done()
})

handler(new HttpRequestMock('/__run__'), response, nextSpy)

mockReporter.write('result')
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 0})
})

it('should set the empty to 1 if successful tests', (done) => {
capturedBrowsers.add(new Browser())
sinon.stub(capturedBrowsers, 'areAllReady', () => true)

response.once('end', () => {
expect(nextSpy).to.not.have.been.called
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
done()
})

handler(new HttpRequestMock('/__run__'), response, nextSpy)

mockReporter.write('result')
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 3, failed: 0})
})

it('should set the empty to 1 if failed tests', (done) => {
capturedBrowsers.add(new Browser())
sinon.stub(capturedBrowsers, 'areAllReady', () => true)

response.once('end', () => {
expect(nextSpy).to.not.have.been.called
expect(response).to.beServedAs(200, 'result\x1FEXIT10')
done()
})

handler(new HttpRequestMock('/__run__'), response, nextSpy)

mockReporter.write('result')
emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 6})
})

it('should not run if there is no browser captured', (done) => {
sinon.stub(fileListMock, 'refresh')

Expand Down
23 changes: 18 additions & 5 deletions test/unit/runner.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ describe('runner', () => {
var EXIT = constant.EXIT_CODE

it('should return 0 exit code if present in the buffer', () => {
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}0`))).to.equal(0)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`))).to.equal(0)
})

it('should null the exit code part of the buffer', () => {
var buffer = new Buffer(`some${EXIT}1`)
var buffer = new Buffer(`some${EXIT}01`)
m.parseExitCode(buffer)

expect(buffer.toString()).to.equal('some\0\0\0\0\0\0')
expect(buffer.toString()).to.equal('some\0\0\0\0\0\0\0')
})

it('should not touch buffer without exit code and return default', () => {
Expand All @@ -41,8 +41,21 @@ describe('runner', () => {
})

it('should parse any single digit exit code', () => {
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}1`))).to.equal(1)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}7`))).to.equal(7)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`))).to.equal(1)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`))).to.equal(7)
})

it('should return exit code 0 if failOnEmptyTestSuite is false and and non-empty int is 0', () => {
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, false)).to.equal(0)
})

it('should return exit code if failOnEmptyTestSuite is true', () => {
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}00`), undefined, true)).to.equal(0)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, true)).to.equal(1)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}07`), undefined, true)).to.equal(7)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`), undefined, true)).to.equal(0)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}11`), undefined, true)).to.equal(1)
expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`), undefined, true)).to.equal(7)
})
})
})

0 comments on commit 8004763

Please sign in to comment.