diff --git a/lib/runner.js b/lib/runner.js index 6952efa92c..018c71b1b4 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -876,15 +876,11 @@ Runner.prototype.run = function(fn) { if (self._delay) { self.emit(constants.EVENT_DELAY_END); } - Runner.immediately(function() { - self.emit(constants.EVENT_RUN_BEGIN); - }); + self.emit(constants.EVENT_RUN_BEGIN); self.runSuite(rootSuite, function() { debug('finished running'); - Runner.immediately(function() { - self.emit(constants.EVENT_RUN_END); - }); + self.emit(constants.EVENT_RUN_END); }); } diff --git a/test/assertions.js b/test/assertions.js index fac66ce94a..64ad548c91 100644 --- a/test/assertions.js +++ b/test/assertions.js @@ -100,6 +100,24 @@ exports.mixinMochaAssertions = function(expect) { code: expect.it('to be greater than', 0) }); }) + .addAssertion( + ' [not] to have failed (with|having) output ', + function(expect, result, output) { + expect(result, '[not] to satisfy', { + code: expect.it('to be greater than', 0), + output: output + }); + } + ) + .addAssertion( + ' [not] to have passed (with|having) output ', + function(expect, result, output) { + expect(result, '[not] to satisfy', { + code: 0, + output: output + }); + } + ) .addAssertion(' [not] to have test count ', function( expect, result, diff --git a/test/integration/events.spec.js b/test/integration/events.spec.js index ef3aea7931..d66109f878 100644 --- a/test/integration/events.spec.js +++ b/test/integration/events.spec.js @@ -75,4 +75,23 @@ describe('event order', function() { }); }); }); + + describe('--retries and --bail test case', function() { + it('should assert --retries event order', function(done) { + runMochaJSON( + 'runner/events-bail-retries.fixture.js', + ['--retries', '1', '--bail'], + function(err, res) { + if (err) { + done(err); + return; + } + expect(res, 'to have failed with error', 'error test A') + .and('to have failed test count', 1) + .and('to have passed test count', 0); + done(); + } + ); + }); + }); }); diff --git a/test/integration/fixtures/runner/events-bail-retries.fixture.js b/test/integration/fixtures/runner/events-bail-retries.fixture.js new file mode 100644 index 0000000000..5ef61ae89a --- /dev/null +++ b/test/integration/fixtures/runner/events-bail-retries.fixture.js @@ -0,0 +1,57 @@ +'use strict'; +var Runner = require('../../../../lib/runner.js'); +var assert = require('assert'); +var constants = Runner.constants; +var EVENT_HOOK_BEGIN = constants.EVENT_HOOK_BEGIN; +var EVENT_HOOK_END = constants.EVENT_HOOK_END; +var EVENT_RUN_BEGIN = constants.EVENT_RUN_BEGIN; +var EVENT_RUN_END = constants.EVENT_RUN_END; +var EVENT_SUITE_BEGIN = constants.EVENT_SUITE_BEGIN; +var EVENT_SUITE_END = constants.EVENT_SUITE_END; +var EVENT_TEST_BEGIN = constants.EVENT_TEST_BEGIN; +var EVENT_TEST_END = constants.EVENT_TEST_END; +var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; +var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; + +var emitOrder = [ + EVENT_RUN_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_RETRY, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_BEGIN, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_TEST_FAIL, + EVENT_TEST_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_HOOK_BEGIN, + EVENT_HOOK_END, + EVENT_SUITE_END, + EVENT_SUITE_END, + EVENT_RUN_END +]; + +var realEmit = Runner.prototype.emit; +Runner.prototype.emit = function(event, ...args) { + // console.log(`emit: ${event}`); + assert.strictEqual(event, emitOrder.shift()); + return realEmit.call(this, event, ...args); +}; + +describe('suite A', function() { + before('before', function() {}); + beforeEach('beforeEach', function() {}); + it('test A', function() { + throw new Error('error test A'); + }); + afterEach('afterEach', function() {}); + after('after', function() {}); +}); diff --git a/test/integration/fixtures/runner/events-bail.fixture.js b/test/integration/fixtures/runner/events-bail.fixture.js index 7b060bd43c..ccb54a0631 100644 --- a/test/integration/fixtures/runner/events-bail.fixture.js +++ b/test/integration/fixtures/runner/events-bail.fixture.js @@ -13,9 +13,9 @@ var EVENT_TEST_END = constants.EVENT_TEST_END; var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var emitOrder = [ - EVENT_SUITE_BEGIN, // incorrect order EVENT_RUN_BEGIN, EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, EVENT_HOOK_BEGIN, EVENT_HOOK_END, EVENT_TEST_BEGIN, diff --git a/test/integration/fixtures/runner/events-basic.fixture.js b/test/integration/fixtures/runner/events-basic.fixture.js index 7a352379ec..6f38878c17 100644 --- a/test/integration/fixtures/runner/events-basic.fixture.js +++ b/test/integration/fixtures/runner/events-basic.fixture.js @@ -18,9 +18,9 @@ var EVENT_TEST_PENDING = constants.EVENT_TEST_PENDING; var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; var emitOrder = [ - EVENT_SUITE_BEGIN, // incorrect order EVENT_RUN_BEGIN, EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, EVENT_HOOK_BEGIN, EVENT_HOOK_END, EVENT_TEST_BEGIN, diff --git a/test/integration/fixtures/runner/events-delay.fixture.js b/test/integration/fixtures/runner/events-delay.fixture.js index 9a50760884..b1ac40ff30 100644 --- a/test/integration/fixtures/runner/events-delay.fixture.js +++ b/test/integration/fixtures/runner/events-delay.fixture.js @@ -20,9 +20,9 @@ var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; var emitOrder = [ EVENT_DELAY_BEGIN, EVENT_DELAY_END, - EVENT_SUITE_BEGIN, // incorrect order EVENT_RUN_BEGIN, EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, EVENT_HOOK_BEGIN, EVENT_HOOK_END, EVENT_TEST_BEGIN, diff --git a/test/integration/fixtures/runner/events-retries.fixture.js b/test/integration/fixtures/runner/events-retries.fixture.js index f8936aebfa..987c3d8162 100644 --- a/test/integration/fixtures/runner/events-retries.fixture.js +++ b/test/integration/fixtures/runner/events-retries.fixture.js @@ -14,9 +14,9 @@ var EVENT_TEST_FAIL = constants.EVENT_TEST_FAIL; var EVENT_TEST_RETRY = constants.EVENT_TEST_RETRY; var emitOrder = [ - EVENT_SUITE_BEGIN, // incorrect order EVENT_RUN_BEGIN, EVENT_SUITE_BEGIN, + EVENT_SUITE_BEGIN, EVENT_HOOK_BEGIN, EVENT_HOOK_END, EVENT_TEST_BEGIN, diff --git a/test/integration/helpers.js b/test/integration/helpers.js index b2e98434c7..0bbaa87e69 100644 --- a/test/integration/helpers.js +++ b/test/integration/helpers.js @@ -81,27 +81,34 @@ module.exports = { var path; path = resolveFixturePath(fixturePath); - args = args || []; + args = (args || []).concat(['--reporter', 'json', path]); return invokeMocha( - args.concat(['--reporter', 'json', path]), + args, function(err, res) { - if (err) return fn(err); + if (err) { + return fn(err); + } + var result; try { - var result = toJSONRunResult(res); - fn(null, result); + // attempt to catch a JSON parsing error *only* here. + // previously, the callback was called within this `try` block, + // which would result in errors thrown from the callback + // getting caught by the `catch` block below. + result = toJSONRunResult(res); } catch (err) { - fn( + return fn( new Error( format( - 'Failed to parse JSON reporter output.\nArgs: %O\nResult:\n\n%O', - args, + 'Failed to parse JSON reporter output: %s\n\nResult:%O', + err, res ) ) ); } + fn(null, result); }, opts ); diff --git a/test/integration/options/compilers.spec.js b/test/integration/options/compilers.spec.js index f3b60a142e..99fa79a7f5 100644 --- a/test/integration/options/compilers.spec.js +++ b/test/integration/options/compilers.spec.js @@ -4,20 +4,18 @@ var helpers = require('../helpers'); var invokeMocha = helpers.invokeMocha; describe('--compilers', function() { - var args = []; + it('should report deprecation', function(done) { + invokeMocha( + ['--compilers', 'coffee:coffee-script/register'], + function(err, res) { + if (err) { + return done(err); + } - before(function() { - args = ['--compilers', 'coffee:coffee-script/register']; - }); - - it('should fail', function(done) { - invokeMocha(args, function(err, res) { - if (err) { - return done(err); - } - - expect(res.code, 'to be', 1); - done(); - }); + expect(res, 'to have failed with output', /compilers is deprecated/i); + done(); + }, + 'pipe' + ); }); }); diff --git a/test/integration/options/grep.spec.js b/test/integration/options/grep.spec.js index 26750a353e..f7faca5bf8 100644 --- a/test/integration/options/grep.spec.js +++ b/test/integration/options/grep.spec.js @@ -1,21 +1,14 @@ 'use strict'; -var path = require('path').posix; var helpers = require('../helpers'); var runMocha = helpers.runMocha; var runMochaJSON = helpers.runMochaJSON; -describe('--grep', function() { - var args = []; - var fixture = path.join('options', 'grep'); - - afterEach(function() { - args = []; - }); +var FIXTURE = 'options/grep'; +describe('--grep', function() { it('should run specs matching a string', function(done) { - args = ['--grep', 'match']; - runMochaJSON(fixture, args, function(err, res) { + runMochaJSON(FIXTURE, ['--grep', 'match'], function(err, res) { if (err) { return done(err); } @@ -27,9 +20,8 @@ describe('--grep', function() { }); describe('should run specs matching a RegExp', function() { - it('with RegExp-like strings (pattern follow by flag)', function(done) { - args = ['--grep', '/match/i']; - runMochaJSON(fixture, args, function(err, res) { + it('with RegExp-like strings (pattern followed by flag)', function(done) { + runMochaJSON(FIXTURE, ['--grep', '/match/i'], function(err, res) { if (err) { return done(err); } @@ -41,8 +33,7 @@ describe('--grep', function() { }); it('with string as pattern', function(done) { - args = ['--grep', '.*']; - runMochaJSON(fixture, args, function(err, res) { + runMochaJSON(FIXTURE, ['--grep', '.*'], function(err, res) { if (err) { return done(err); } @@ -55,10 +46,9 @@ describe('--grep', function() { }); }); - describe('when --invert used', function() { + describe('when used with --invert', function() { it('should run specs that do not match the pattern', function(done) { - args = ['--grep', 'fail', '--invert']; - runMochaJSON(fixture, args, function(err, res) { + runMochaJSON(FIXTURE, ['--grep', 'fail', '--invert'], function(err, res) { if (err) { return done(err); } @@ -68,40 +58,45 @@ describe('--grep', function() { done(); }); }); + }); - it('should throw an error when option used in isolation', function(done) { - var spawnOpts = {stdio: 'pipe'}; - args = ['--invert']; + describe('when both --fgrep and --grep used together', function() { + it('should report an error', function(done) { runMocha( - fixture, - args, + FIXTURE, + ['--fgrep', 'first', '--grep', 'second'], function(err, res) { if (err) { return done(err); } - expect(res, 'to satisfy', { - code: 1, - output: /--invert.*--grep / - }); + expect(res, 'to have failed with output', /mutually exclusive/i); done(); }, - spawnOpts + 'pipe' ); }); }); +}); - describe('when both --fgrep and --grep used together', function() { - it('should conflict', function(done) { - // var fixture = 'uncaught.fixture.js'; - args = ['--fgrep', 'first', '--grep', 'second']; - - runMocha(fixture, args, function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have failed'); - done(); - }); +describe('--invert', function() { + describe('when used without --fgrep or --grep', function() { + it('it should report an error', function(done) { + runMocha( + FIXTURE, + ['--invert'], + function(err, res) { + if (err) { + return done(err); + } + expect( + res, + 'to have failed with output', + /--invert.*--grep / + ); + done(); + }, + 'pipe' + ); }); }); });