From 57930a290c53f2d9b5f2be9797a08062b913f552 Mon Sep 17 00:00:00 2001 From: "P. Roebuck" Date: Thu, 24 Jan 2019 16:51:12 -0600 Subject: [PATCH] Reorder cmdline options top-level test suites lexically (#3674) * refactor(integration/options.spec.js): Reorder top-level test suites lexically Reordered each cmdline option's test suite alphabetically. No code changes. * test(integration/options): Break out tests into option-specific spec files Made individual spec files for various options from the original. * test(options/opts.spec.js): Fix issue on Windows [skip travis] Removed cwd setting as (hopefully) unneeded since the test deals with generating error from nonexistent file. * test(options/help.spec.js): Windows bughunt [skip travis][skip netlify] * test(options/help.spec.js): Fix spawnOpts cwd Must use platform-specific `path.join` for `spawnOpts.cwd`, or it fails on Windows. * test: Modify describe titles for consistency Updated a couple suite descriptions for consistency. --- test/integration/options.spec.js | 863 +----------------- test/integration/options/asyncOnly.spec.js | 37 + test/integration/options/bail.spec.js | 128 +++ test/integration/options/compilers.spec.js | 23 + test/integration/options/delay.spec.js | 58 ++ test/integration/options/exclude.spec.js | 75 ++ test/integration/options/exit.spec.js | 68 ++ test/integration/options/extension.spec.js | 31 + test/integration/options/file.spec.js | 55 ++ test/integration/options/forbidOnly.spec.js | 127 +++ .../integration/options/forbidPending.spec.js | 91 ++ test/integration/options/grep.spec.js | 107 +++ test/integration/options/help.spec.js | 26 + test/integration/options/opts.spec.js | 46 + test/integration/options/retries.spec.js | 24 + test/integration/options/sort.spec.js | 28 + test/integration/options/watch.spec.js | 56 ++ 17 files changed, 989 insertions(+), 854 deletions(-) create mode 100644 test/integration/options/asyncOnly.spec.js create mode 100644 test/integration/options/bail.spec.js create mode 100644 test/integration/options/compilers.spec.js create mode 100644 test/integration/options/delay.spec.js create mode 100644 test/integration/options/exclude.spec.js create mode 100644 test/integration/options/exit.spec.js create mode 100644 test/integration/options/extension.spec.js create mode 100644 test/integration/options/file.spec.js create mode 100644 test/integration/options/forbidOnly.spec.js create mode 100644 test/integration/options/forbidPending.spec.js create mode 100644 test/integration/options/grep.spec.js create mode 100644 test/integration/options/help.spec.js create mode 100644 test/integration/options/opts.spec.js create mode 100644 test/integration/options/retries.spec.js create mode 100644 test/integration/options/sort.spec.js create mode 100644 test/integration/options/watch.spec.js diff --git a/test/integration/options.spec.js b/test/integration/options.spec.js index 6854528566..f3336c0ff1 100644 --- a/test/integration/options.spec.js +++ b/test/integration/options.spec.js @@ -1,861 +1,16 @@ 'use strict'; +/** + * Tests for cmdline options. + * Provides umbrella for all spec files in the "options" subdirectory. + */ + var path = require('path'); -var helpers = require('./helpers'); -var runMocha = helpers.runMocha; -var runMochaJSON = helpers.runMochaJSON; -var runMochaJSONRaw = helpers.runMochaJSONRaw; -var invokeMocha = helpers.invokeMocha; -var resolvePath = helpers.resolveFixturePath; -var toJSONRunResult = helpers.toJSONRunResult; -var args = []; +var utils = require('../../lib/utils'); describe('options', function() { - describe('--async-only', function() { - before(function() { - args = ['--async-only']; - }); - - it('should fail synchronous specs', function(done) { - runMochaJSON('options/async-only-sync.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed'); - done(); - }); - }); - - it('should allow asynchronous specs', function(done) { - runMochaJSON('options/async-only-async.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed'); - done(); - }); - }); - }); - - describe('--bail', function() { - before(function() { - args = ['--bail']; - }); - - it('should stop after the first error', function(done) { - runMochaJSON('options/bail.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - - expect(res, 'to have failed') - .and('to have passed test', 'should display this spec') - .and('to have failed test', 'should only display this error') - .and('to have passed test count', 1) - .and('to have failed test count', 1); - done(); - }); - }); - - it('should stop after the first error - async', function(done) { - runMochaJSON('options/bail-async.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - - expect(res, 'to have failed') - .and('to have passed test', 'should display this spec') - .and('to have failed test', 'should only display this error') - .and('to have passed test count', 1) - .and('to have failed test count', 1); - done(); - }); - }); - - it('should stop all tests after failing "before" hook', function(done) { - runMochaJSON('options/bail-with-before.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have failed test count', 1) - .and('to have failed test', '"before all" hook: before suite1') - .and('to have passed test count', 0); - done(); - }); - }); - - it('should stop all tests after failing "beforeEach" hook', function(done) { - runMochaJSON('options/bail-with-beforeEach.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have failed test count', 1) - .and( - 'to have failed test', - '"before each" hook: beforeEach suite1 for "test suite1"' - ) - .and('to have passed test count', 0); - done(); - }); - }); - - it('should stop all tests after failing test', function(done) { - runMochaJSON('options/bail-with-test.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have failed test count', 1) - .and('to have failed test', 'test suite1') - .and('to have passed test count', 0); - done(); - }); - }); - - it('should stop all tests after failing "after" hook', function(done) { - runMochaJSON('options/bail-with-after.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have failed test count', 1) - .and('to have failed test', '"after all" hook: after suite1A') - .and('to have passed test count', 2) - .and('to have passed test order', 'test suite1', 'test suite1A'); - done(); - }); - }); - - it('should stop all tests after failing "afterEach" hook', function(done) { - runMochaJSON('options/bail-with-afterEach.fixture.js', args, function( - err, - res - ) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have failed test count', 1) - .and( - 'to have failed test', - '"after each" hook: afterEach suite1A for "test suite1A"' - ) - .and('to have passed test count', 2) - .and('to have passed test order', 'test suite1', 'test suite1A'); - done(); - }); - }); - }); - - describe('--sort', function() { - before(function() { - args = ['--sort']; - }); - - it('should sort tests in alphabetical order', function(done) { - runMochaJSON('options/sort*', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed test count', 2).and( - 'to have passed test order', - 'should be executed first' - ); - done(); - }); - }); - }); - - describe('--file', function() { - it('should run tests passed via file first', function(done) { - args = ['--file', resolvePath('options/file-alpha.fixture.js')]; - - runMochaJSON('options/file-beta.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 2) - .and('to have passed test order', 'should be executed first'); - done(); - }); - }); - - it('should run multiple tests passed via file first', function(done) { - args = [ - '--file', - resolvePath('options/file-alpha.fixture.js'), - '--file', - resolvePath('options/file-beta.fixture.js') - ]; - - runMochaJSON('options/file-theta.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 3) - .and( - 'to have passed test order', - 'should be executed first', - 'should be executed second', - 'should be executed third' - ); - done(); - }); - }); - }); - - describe('--delay', function() { - before(function() { - args = ['--delay']; - }); - - it('should run the generated test suite', function(done) { - runMochaJSON('options/delay.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed').and('to have passed test count', 1); - done(); - }); - }); - - it('should execute exclusive tests only', function(done) { - runMochaJSON('options/delay-only.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 2) - .and( - 'to have passed test order', - 'should run this', - 'should run this, too' - ); - done(); - }); - }); - - it('should throw an error if the test suite failed to run', function(done) { - runMochaJSON('options/delay-fail.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed').and( - 'to have failed test', - 'Uncaught error outside test suite' - ); - done(); - }); - }); - }); - - describe('--grep', function() { - it('runs specs matching a string', function(done) { - args = ['--grep', 'match']; - runMochaJSON('options/grep.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 2) - .and('not to have pending tests'); - done(); - }); - }); - - describe('runs specs matching a RegExp', function() { - it('with RegExp like strings(pattern follow by flag)', function(done) { - args = ['--grep', '/match/i']; - runMochaJSON('options/grep.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 4) - .and('not to have pending tests'); - done(); - }); - }); - - it('string as pattern', function(done) { - args = ['--grep', '.*']; - runMochaJSON('options/grep.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('to have passed test count', 4) - .and('to have failed test count', 1) - .and('not to have pending tests'); - done(); - }); - }); - }); - - describe('with --invert', function() { - it('runs specs that do not match the pattern', function(done) { - args = ['--grep', 'fail', '--invert']; - runMochaJSON('options/grep.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed') - .and('to have passed test count', 4) - .and('not to have pending tests'); - done(); - }); - }); - - it('should throw an error when `--invert` used in isolation', function(done) { - args = ['--invert']; - runMocha( - 'options/grep.fixture.js', - args, - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to satisfy', { - code: 1, - output: /--invert.*--grep / - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - }); - }); - - describe('--retries', function() { - it('retries after a certain threshold', function(done) { - args = ['--retries', '3']; - runMochaJSON('options/retries.fixture.js', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed') - .and('not to have pending tests') - .and('not to have passed tests') - .and('to have retried test', 'should fail', 3); - done(); - }); - }); - }); - - describe('--forbid-only', function() { - var onlyErrorMessage = '`.only` forbidden'; - - before(function() { - args = ['--forbid-only']; - }); - - it('succeeds if there are only passed tests', function(done) { - runMochaJSON('options/forbid-only/passed', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed'); - done(); - }); - }); - - it('fails if there are tests marked only', function(done) { - runMochaJSON('options/forbid-only/only', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed with error', onlyErrorMessage); - done(); - }); - }); - - it('fails if there are tests in suites marked only', function(done) { - runMocha( - 'options/forbid-only/only-suite', - args, - function(err, res) { - if (err) { - done(err); - return; - } - - expect(res, 'to satisfy', { - code: 1, - output: new RegExp(onlyErrorMessage) - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - - it('fails if there is empty suite marked only', function(done) { - runMocha( - 'options/forbid-only/only-empty-suite', - args, - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to satisfy', { - code: 1, - output: new RegExp(onlyErrorMessage) - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - - it('fails if there is suite marked only which matches a grep', function(done) { - runMocha( - 'options/forbid-only/only-suite', - args.concat('--fgrep', 'suite marked with only'), - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to satisfy', { - code: 1, - output: new RegExp(onlyErrorMessage) - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - - it('succeeds if suite marked only does not match grep', function(done) { - runMochaJSON( - 'options/forbid-only/only-suite', - args.concat('--fgrep', 'bumble bees'), - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed'); - done(); - } - ); - }); - - it('succeeds if suite marked only does not match grep (using "invert")', function(done) { - runMochaJSON( - 'options/forbid-only/only-suite', - args.concat('--fgrep', 'suite marked with only', '--invert'), - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed'); - done(); - } - ); - }); - }); - - describe('--forbid-pending', function() { - var pendingErrorMessage = 'Pending test forbidden'; - - before(function() { - args = ['--forbid-pending']; - }); - - it('succeeds if there are only passed tests', function(done) { - runMochaJSON('options/forbid-pending/passed', args, function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have passed'); - done(); - }); - }); - - it('fails if there are tests in suites marked skip', function(done) { - runMocha( - 'options/forbid-pending/skip-suite', - args, - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to satisfy', { - code: 1, - output: new RegExp(pendingErrorMessage) - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - - it('fails if there is empty suite marked pending', function(done) { - runMocha( - 'options/forbid-pending/skip-empty-suite', - args, - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to satisfy', { - code: 1, - output: new RegExp(pendingErrorMessage) - }); - done(); - }, - {stdio: 'pipe'} - ); - }); - - var forbidPendingFailureTests = { - 'fails if there are tests marked skip': 'skip', - 'fails if there are pending tests': 'pending', - 'fails if tests call `skip()`': 'this-skip', - 'fails if beforeEach calls `skip()`': 'beforeEach-this-skip', - 'fails if before calls `skip()`': 'before-this-skip' - }; - - Object.keys(forbidPendingFailureTests).forEach(function(title) { - it(title, function(done) { - runMochaJSON( - path.join( - 'options', - 'forbid-pending', - forbidPendingFailureTests[title] - ), - args, - function(err, res) { - if (err) { - done(err); - return; - } - expect(res, 'to have failed with error', pendingErrorMessage); - done(); - } - ); - }); - }); - }); - - describe('--exit', function() { - var behaviors = { - enabled: '--exit', - disabled: '--no-exit' - }; - - /** - * Returns a test that executes Mocha in a subprocess with either - * `--exit`, `--no-exit`, or default behavior. - * @param {boolean} shouldExit - Expected result; `true` if Mocha should - * have force-killed the process. - * @param {string} [behavior] - 'enabled' or 'disabled' - * @returns {Function} - */ - var runExit = function(shouldExit, behavior) { - return function(done) { - var timeout = this.timeout(); - this.timeout(0); - this.slow(Infinity); - var didExit = true; - var t; - var args = behaviors[behavior] ? [behaviors[behavior]] : []; - - var mocha = runMochaJSON('exit.fixture.js', args, function(err) { - clearTimeout(t); - if (err) { - done(err); - return; - } - expect(didExit, 'to be', shouldExit); - done(); - }); - - // if this callback happens, then Mocha didn't automatically exit. - t = setTimeout(function() { - didExit = false; - // this is the only way to kill the child, afaik. - // after the process ends, the callback to `run()` above is handled. - mocha.kill('SIGINT'); - }, timeout - 500); - }; - }; - - describe('default behavior', function() { - it('should force exit after root suite completion', runExit(false)); - }); - - describe('with exit enabled', function() { - it( - 'should force exit after root suite completion', - runExit(true, 'enabled') - ); - }); - - describe('with exit disabled', function() { - it( - 'should not force exit after root suite completion', - runExit(false, 'disabled') - ); - }); - }); - - describe('--help', function() { - it('works despite the presence of mocha.opts', function(done) { - invokeMocha( - ['-h'], - function(error, result) { - if (error) { - return done(error); - } - expect(result.output, 'to contain', 'Run tests with Mocha'); - done(); - }, - {cwd: path.join(__dirname, 'fixtures', 'options', 'help')} - ); - }); - }); - - describe('--opts', function() { - var testFile = path.join('options', 'opts.fixture.js'); - - it('works despite nonexistent default options file', function(done) { - args = []; - runMochaJSON(testFile, args, function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to have passed').and('to have passed test count', 1); - return done(); - }); - }); - - it('should throw an error due to nonexistent options file', function(done) { - args = ['--opts', 'nosuchoptionsfile', testFile]; - invokeMocha( - args, - function(err, res) { - if (err) { - return done(err); - } - expect(res, 'to satisfy', { - code: 1, - output: /unable to read nosuchoptionsfile/i - }); - return done(); - }, - {cwd: path.join(__dirname, 'fixtures'), stdio: 'pipe'} - ); - }); - }); - - describe('--exclude', function() { - /* - * Runs mocha in {path} with the given args. - * Calls handleResult with the result. - */ - function runMochaTest(fixture, args, handleResult, done) { - runMochaJSON(fixture, args, function(err, res) { - if (err) { - done(err); - return; - } - handleResult(res); - done(); - }); - } - - it('should exclude specific files', function(done) { - runMochaTest( - 'options/exclude/*.fixture.js', - [ - '--exclude', - 'test/integration/fixtures/options/exclude/fail.fixture.js' - ], - function(res) { - expect(res, 'to have passed') - .and('to have run test', 'should find this test') - .and('not to have pending tests'); - }, - done - ); - }); - - it('should exclude globbed files', function(done) { - runMochaTest( - 'options/exclude/**/*.fixture.js', - ['--exclude', '**/fail.fixture.js'], - function(res) { - expect(res, 'to have passed') - .and('not to have pending tests') - .and('to have passed test count', 2); - }, - done - ); - }); - - it('should exclude multiple patterns', function(done) { - runMochaTest( - 'options/exclude/**/*.fixture.js', - [ - '--exclude', - 'test/integration/fixtures/options/exclude/fail.fixture.js', - '--exclude', - 'test/integration/fixtures/options/exclude/nested/fail.fixture.js' - ], - function(res) { - expect(res, 'to have passed') - .and('not to have pending tests') - .and('to have passed test count', 2); - }, - done - ); - }); - }); - - if (process.platform !== 'win32') { - // Windows: Feature works but SIMULATING the signal (ctr+c), via child process, does not work - // due to lack of *nix signal compliance. - describe('--watch', function() { - describe('with watch enabled', function() { - it('should show the cursor and signal correct exit code, when watch process is terminated', function(done) { - this.timeout(0); - this.slow(3000); - // executes Mocha in a subprocess - var mocha = runMochaJSONRaw( - 'exit.fixture.js', - ['--watch'], - function(err, data) { - // After the process ends, this callback is ran - if (err) { - done(err); - return; - } - - var expectedCloseCursor = '\u001b[?25h'; - expect(data.output, 'to contain', expectedCloseCursor); - expect(data.code, 'to be', 130); - done(); - }, - {stdio: 'pipe'} - ); - setTimeout(function() { - // kill the child process - mocha.kill('SIGINT'); - }, 1000); - }); - }); - }); - } - - describe('--compilers', function() { - it('should fail', function(done) { - invokeMocha(['--compilers', 'coffee:coffee-script/register'], function( - error, - result - ) { - if (error) { - return done(error); - } - expect(result.code, 'to be', 1); - done(); - }); - }); - }); - - describe('--fgrep and --grep', function() { - it('should conflict', function(done) { - runMocha( - 'uncaught.fixture.js', - ['--fgrep', 'first', '--grep', 'second'], - function(err, result) { - if (err) { - return done(err); - } - expect(result, 'to have failed'); - done(); - } - ); - }); - }); + var specDir = path.join(__dirname, 'options'); + var specs = utils.lookupFiles(specDir, ['js']).sort(); - describe('--extension', function() { - it('should allow comma-separated variables', function(done) { - invokeMocha( - [ - '--require', - 'coffee-script/register', - '--require', - './test/setup', - '--reporter', - 'json', - '--extension', - 'js,coffee', - 'test/integration/fixtures/options/extension' - ], - function(err, result) { - if (err) { - return done(err); - } - expect(toJSONRunResult(result), 'to have passed').and( - 'to have passed test count', - 2 - ); - done(); - } - ); - }); - }); + specs.forEach(require); }); diff --git a/test/integration/options/asyncOnly.spec.js b/test/integration/options/asyncOnly.spec.js new file mode 100644 index 0000000000..d9e3e8e064 --- /dev/null +++ b/test/integration/options/asyncOnly.spec.js @@ -0,0 +1,37 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--async-only', function() { + var args = []; + + before(function() { + args = ['--async-only']; + }); + + it('should fail synchronous specs', function(done) { + var fixture = path.join('options', 'async-only-sync'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed'); + done(); + }); + }); + + it('should allow asynchronous specs', function(done) { + var fixture = path.join('options', 'async-only-async'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed'); + done(); + }); + }); +}); diff --git a/test/integration/options/bail.spec.js b/test/integration/options/bail.spec.js new file mode 100644 index 0000000000..067cba7890 --- /dev/null +++ b/test/integration/options/bail.spec.js @@ -0,0 +1,128 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--bail', function() { + var args = []; + + before(function() { + args = ['--bail']; + }); + + it('should stop after the first error', function(done) { + var fixture = path.join('options', 'bail'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have passed test', 'should display this spec') + .and('to have failed test', 'should only display this error') + .and('to have passed test count', 1) + .and('to have failed test count', 1); + done(); + }); + }); + + it('should stop after the first error - async', function(done) { + var fixture = path.join('options', 'bail-async'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have passed test', 'should display this spec') + .and('to have failed test', 'should only display this error') + .and('to have passed test count', 1) + .and('to have failed test count', 1); + done(); + }); + }); + + it('should stop all tests after failing "before" hook', function(done) { + var fixture = path.join('options', 'bail-with-before'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have failed test count', 1) + .and('to have failed test', '"before all" hook: before suite1') + .and('to have passed test count', 0); + done(); + }); + }); + + it('should stop all tests after failing "beforeEach" hook', function(done) { + var fixture = path.join('options', 'bail-with-beforeEach'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have failed test count', 1) + .and( + 'to have failed test', + '"before each" hook: beforeEach suite1 for "test suite1"' + ) + .and('to have passed test count', 0); + done(); + }); + }); + + it('should stop all tests after failing test', function(done) { + var fixture = path.join('options', 'bail-with-test'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have failed test count', 1) + .and('to have failed test', 'test suite1') + .and('to have passed test count', 0); + done(); + }); + }); + + it('should stop all tests after failing "after" hook', function(done) { + var fixture = path.join('options', 'bail-with-after'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have failed test count', 1) + .and('to have failed test', '"after all" hook: after suite1A') + .and('to have passed test count', 2) + .and('to have passed test order', 'test suite1', 'test suite1A'); + done(); + }); + }); + + it('should stop all tests after failing "afterEach" hook', function(done) { + var fixture = path.join('options', 'bail-with-afterEach'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed') + .and('to have failed test count', 1) + .and( + 'to have failed test', + '"after each" hook: afterEach suite1A for "test suite1A"' + ) + .and('to have passed test count', 2) + .and('to have passed test order', 'test suite1', 'test suite1A'); + done(); + }); + }); +}); diff --git a/test/integration/options/compilers.spec.js b/test/integration/options/compilers.spec.js new file mode 100644 index 0000000000..f3b60a142e --- /dev/null +++ b/test/integration/options/compilers.spec.js @@ -0,0 +1,23 @@ +'use strict'; + +var helpers = require('../helpers'); +var invokeMocha = helpers.invokeMocha; + +describe('--compilers', function() { + var args = []; + + before(function() { + args = ['--compilers', 'coffee:coffee-script/register']; + }); + + it('should fail', function(done) { + invokeMocha(args, function(err, res) { + if (err) { + return done(err); + } + + expect(res.code, 'to be', 1); + done(); + }); + }); +}); diff --git a/test/integration/options/delay.spec.js b/test/integration/options/delay.spec.js new file mode 100644 index 0000000000..ae9ff70f29 --- /dev/null +++ b/test/integration/options/delay.spec.js @@ -0,0 +1,58 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--delay', function() { + var args = []; + + before(function() { + args = ['--delay']; + }); + + it('should run the generated test suite', function(done) { + var fixture = path.join('options', 'delay'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed').and('to have passed test count', 1); + done(); + }); + }); + + it('should execute exclusive tests only', function(done) { + var fixture = path.join('options', 'delay-only'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed') + .and('to have passed test count', 2) + .and( + 'to have passed test order', + 'should run this', + 'should run this, too' + ); + done(); + }); + }); + + it('should throw an error if the test suite failed to run', function(done) { + var fixture = path.join('options', 'delay-fail'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have failed').and( + 'to have failed test', + 'Uncaught error outside test suite' + ); + done(); + }); + }); +}); diff --git a/test/integration/options/exclude.spec.js b/test/integration/options/exclude.spec.js new file mode 100644 index 0000000000..dc56410c1f --- /dev/null +++ b/test/integration/options/exclude.spec.js @@ -0,0 +1,75 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; +var resolvePath = helpers.resolveFixturePath; + +describe('--exclude', function() { + /* + * Runs mocha in {path} with the given args. + * Calls handleResult with the result. + * + * @param {string} fixture + * @param {string[]} args + * @param {function} handleResult + * @param {function} done + */ + function runMochaTest(fixture, args, handleResult, done) { + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + handleResult(res); + done(); + }); + } + + it('should exclude specific files', function(done) { + var fixtures = path.join('options', 'exclude', '*'); + runMochaTest( + fixtures, + ['--exclude', resolvePath(path.join('options', 'exclude', 'fail'))], + function(res) { + expect(res, 'to have passed') + .and('to have run test', 'should find this test') + .and('not to have pending tests'); + }, + done + ); + }); + + it('should exclude globbed files', function(done) { + var fixtures = path.join('options', 'exclude', '**', '*'); + runMochaTest( + fixtures, + ['--exclude', '**/fail.fixture.js'], + function(res) { + expect(res, 'to have passed') + .and('not to have pending tests') + .and('to have passed test count', 2); + }, + done + ); + }); + + it('should exclude multiple patterns', function(done) { + var fixtures = path.join('options', 'exclude', '**', '*'); + runMochaTest( + fixtures, + [ + '--exclude', + resolvePath(path.join('options', 'exclude', 'fail')), + '--exclude', + resolvePath(path.join('options', 'exclude', 'nested', 'fail')) + ], + function(res) { + expect(res, 'to have passed') + .and('not to have pending tests') + .and('to have passed test count', 2); + }, + done + ); + }); +}); diff --git a/test/integration/options/exit.spec.js b/test/integration/options/exit.spec.js new file mode 100644 index 0000000000..376496f4fc --- /dev/null +++ b/test/integration/options/exit.spec.js @@ -0,0 +1,68 @@ +'use strict'; + +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--exit', function() { + var behaviors = { + enabled: '--exit', + disabled: '--no-exit' + }; + + /** + * Returns a test that executes Mocha in a subprocess with either + * `--exit`, `--no-exit`, or default behavior. + * + * @param {boolean} shouldExit - Expected result; `true` if Mocha should + * have force-killed the process. + * @param {string} [behavior] - 'enabled' or 'disabled' + * @returns {Function} async function implementing the test + */ + var runExit = function(shouldExit, behavior) { + return function(done) { + var timeout = this.timeout(); + this.timeout(0); + this.slow(Infinity); + + var didExit = true; + var timeoutObj; + var fixture = 'exit.fixture.js'; + var args = behaviors[behavior] ? [behaviors[behavior]] : []; + + var mocha = runMochaJSON(fixture, args, function postmortem(err) { + clearTimeout(timeoutObj); + if (err) { + return done(err); + } + expect(didExit, 'to be', shouldExit); + done(); + }); + + // If this callback happens, then Mocha didn't automatically exit. + timeoutObj = setTimeout(function() { + didExit = false; + // This is the only way to kill the child, afaik. + // After the process ends, the callback to `run()` above is handled. + mocha.kill('SIGINT'); + }, timeout - 500); + }; + }; + + describe('default behavior', function() { + it('should force exit after root suite completion', runExit(false)); + }); + + describe('when enabled', function() { + it( + 'should force exit after root suite completion', + runExit(true, 'enabled') + ); + }); + + describe('when disabled', function() { + it( + 'should not force exit after root suite completion', + runExit(false, 'disabled') + ); + }); +}); diff --git a/test/integration/options/extension.spec.js b/test/integration/options/extension.spec.js new file mode 100644 index 0000000000..760e3bcd88 --- /dev/null +++ b/test/integration/options/extension.spec.js @@ -0,0 +1,31 @@ +'use strict'; + +var helpers = require('../helpers'); +var invokeMocha = helpers.invokeMocha; +var toJSONRunResult = helpers.toJSONRunResult; + +describe('--extension', function() { + it('should allow comma-separated variables', function(done) { + var args = [ + '--require', + 'coffee-script/register', + '--require', + './test/setup', + '--reporter', + 'json', + '--extension', + 'js,coffee', + 'test/integration/fixtures/options/extension' + ]; + invokeMocha(args, function(err, res) { + if (err) { + return done(err); + } + expect(toJSONRunResult(res), 'to have passed').and( + 'to have passed test count', + 2 + ); + done(); + }); + }); +}); diff --git a/test/integration/options/file.spec.js b/test/integration/options/file.spec.js new file mode 100644 index 0000000000..f96b58668d --- /dev/null +++ b/test/integration/options/file.spec.js @@ -0,0 +1,55 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; +var resolvePath = helpers.resolveFixturePath; + +describe('--file', function() { + var args = []; + var fixtures = { + alpha: path.join('options', 'file-alpha'), + beta: path.join('options', 'file-beta'), + theta: path.join('options', 'file-theta') + }; + + it('should run tests passed via file first', function(done) { + args = ['--file', resolvePath(fixtures.alpha)]; + + var fixture = fixtures.beta; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have passed test count', 2) + .and('to have passed test order', 'should be executed first'); + done(); + }); + }); + + it('should run multiple tests passed via file first', function(done) { + args = [ + '--file', + resolvePath(fixtures.alpha), + '--file', + resolvePath(fixtures.beta) + ]; + + var fixture = fixtures.theta; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have passed test count', 3) + .and( + 'to have passed test order', + 'should be executed first', + 'should be executed second', + 'should be executed third' + ); + done(); + }); + }); +}); diff --git a/test/integration/options/forbidOnly.spec.js b/test/integration/options/forbidOnly.spec.js new file mode 100644 index 0000000000..592f0f25df --- /dev/null +++ b/test/integration/options/forbidOnly.spec.js @@ -0,0 +1,127 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMocha = helpers.runMocha; +var runMochaJSON = helpers.runMochaJSON; + +describe('--forbid-only', function() { + var args = []; + var onlyErrorMessage = '`.only` forbidden'; + + beforeEach(function() { + args = ['--forbid-only']; + }); + + it('should succeed if there are only passed tests', function(done) { + var fixture = path.join('options', 'forbid-only', 'passed'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed'); + done(); + }); + }); + + it('should fail if there are tests marked only', function(done) { + var fixture = path.join('options', 'forbid-only', 'only'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', onlyErrorMessage); + done(); + }); + }); + + it('should fail if there are tests in suites marked only', function(done) { + var fixture = path.join('options', 'forbid-only', 'only-suite'); + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixture, + args, + function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(onlyErrorMessage) + }); + done(); + }, + spawnOpts + ); + }); + + it('should fail if there is empty suite marked only', function(done) { + var fixture = path.join('options', 'forbid-only', 'only-empty-suite'); + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixture, + args, + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(onlyErrorMessage) + }); + done(); + }, + spawnOpts + ); + }); + + it('should fail if there is suite marked only which matches grep', function(done) { + var fixture = path.join('options', 'forbid-only', 'only-suite'); + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixture, + args.concat('--fgrep', 'suite marked with only'), + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(onlyErrorMessage) + }); + done(); + }, + spawnOpts + ); + }); + + it('should succeed if suite marked only does not match grep', function(done) { + var fixture = path.join('options', 'forbid-only', 'only-suite'); + runMochaJSON(fixture, args.concat('--fgrep', 'bumble bees'), function( + err, + res + ) { + if (err) { + return done(err); + } + expect(res, 'to have passed'); + done(); + }); + }); + + it('should succeed if suite marked only does not match inverted grep', function(done) { + var fixture = path.join('options', 'forbid-only', 'only-suite'); + runMochaJSON( + fixture, + args.concat('--fgrep', 'suite marked with only', '--invert'), + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed'); + done(); + } + ); + }); +}); diff --git a/test/integration/options/forbidPending.spec.js b/test/integration/options/forbidPending.spec.js new file mode 100644 index 0000000000..ab590825a0 --- /dev/null +++ b/test/integration/options/forbidPending.spec.js @@ -0,0 +1,91 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMocha = helpers.runMocha; +var runMochaJSON = helpers.runMochaJSON; + +describe('--forbid-pending', function() { + var args = []; + var pendingErrorMessage = 'Pending test forbidden'; + + before(function() { + args = ['--forbid-pending']; + }); + + it('should succeed if there are only passed tests', function(done) { + var fixture = path.join('options', 'forbid-pending', 'passed'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed'); + done(); + }); + }); + + it('should fail if there are tests in suites marked skip', function(done) { + var fixture = path.join('options', 'forbid-pending', 'skip-suite'); + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixture, + args, + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(pendingErrorMessage) + }); + done(); + }, + spawnOpts + ); + }); + + it('should fail if there is empty suite marked pending', function(done) { + var fixture = path.join('options', 'forbid-pending', 'skip-empty-suite'); + var spawnOpts = {stdio: 'pipe'}; + runMocha( + fixture, + args, + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(pendingErrorMessage) + }); + done(); + }, + spawnOpts + ); + }); + + var forbidPendingFailureTests = { + 'should fail if there are tests marked skip': 'skip', + 'should fail if there are pending tests': 'pending', + 'should fail if tests call `skip()`': 'this-skip', + 'should fail if beforeEach calls `skip()`': 'beforeEach-this-skip', + 'should fail if before calls `skip()`': 'before-this-skip' + }; + + Object.keys(forbidPendingFailureTests).forEach(function(title) { + it(title, function(done) { + var fixture = path.join( + 'options', + 'forbid-pending', + forbidPendingFailureTests[title] + ); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed with error', pendingErrorMessage); + done(); + }); + }); + }); +}); diff --git a/test/integration/options/grep.spec.js b/test/integration/options/grep.spec.js new file mode 100644 index 0000000000..26750a353e --- /dev/null +++ b/test/integration/options/grep.spec.js @@ -0,0 +1,107 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMocha = helpers.runMocha; +var runMochaJSON = helpers.runMochaJSON; + +describe('--grep', function() { + var args = []; + var fixture = path.join('options', 'grep'); + + afterEach(function() { + args = []; + }); + + it('should run specs matching a string', function(done) { + args = ['--grep', 'match']; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have passed test count', 2) + .and('not to have pending tests'); + done(); + }); + }); + + describe('should run specs matching a RegExp', function() { + it('with RegExp-like strings (pattern follow by flag)', function(done) { + args = ['--grep', '/match/i']; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have passed test count', 4) + .and('not to have pending tests'); + done(); + }); + }); + + it('with string as pattern', function(done) { + args = ['--grep', '.*']; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed') + .and('to have passed test count', 4) + .and('to have failed test count', 1) + .and('not to have pending tests'); + done(); + }); + }); + }); + + describe('when --invert used', function() { + it('should run specs that do not match the pattern', function(done) { + args = ['--grep', 'fail', '--invert']; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have passed') + .and('to have passed test count', 4) + .and('not to have pending tests'); + done(); + }); + }); + + it('should throw an error when option used in isolation', function(done) { + var spawnOpts = {stdio: 'pipe'}; + args = ['--invert']; + runMocha( + fixture, + args, + function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to satisfy', { + code: 1, + output: /--invert.*--grep / + }); + done(); + }, + spawnOpts + ); + }); + }); + + describe('when both --fgrep and --grep used together', function() { + it('should conflict', function(done) { + // var fixture = 'uncaught.fixture.js'; + args = ['--fgrep', 'first', '--grep', 'second']; + + runMocha(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed'); + done(); + }); + }); + }); +}); diff --git a/test/integration/options/help.spec.js b/test/integration/options/help.spec.js new file mode 100644 index 0000000000..fe182bce95 --- /dev/null +++ b/test/integration/options/help.spec.js @@ -0,0 +1,26 @@ +'use strict'; + +var path = require('path'); +var helpers = require('../helpers'); +var invokeMocha = helpers.invokeMocha; + +describe('--help', function() { + it('should work despite the presence of "mocha.opts"', function(done) { + var args = ['-h']; + // :NOTE: Must use platform-specific `path.join` for `spawnOpts.cwd` + var fixtureDir = path.join(__dirname, '..', 'fixtures', 'options', 'help'); + var spawnOpts = {cwd: fixtureDir}; + + invokeMocha( + args, + function(err, res) { + if (err) { + return done(err); + } + expect(res.output, 'to contain', 'Run tests with Mocha'); + done(); + }, + spawnOpts + ); + }); +}); diff --git a/test/integration/options/opts.spec.js b/test/integration/options/opts.spec.js new file mode 100644 index 0000000000..51f49fec8b --- /dev/null +++ b/test/integration/options/opts.spec.js @@ -0,0 +1,46 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; +var invokeMocha = helpers.invokeMocha; +var resolvePath = helpers.resolveFixturePath; + +describe('--opts', function() { + var args = []; + var fixture = path.join('options', 'opts'); + + it('should work despite nonexistent default options file', function(done) { + args = []; + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + + expect(res, 'to have passed').and('to have passed test count', 1); + done(); + }); + }); + + it('should throw an error due to nonexistent options file', function(done) { + var spawnOpts = {stdio: 'pipe'}; + var nonexistentFile = 'nosuchoptionsfile'; + args = ['--opts', nonexistentFile, resolvePath(fixture)]; + invokeMocha( + args, + function(err, res) { + if (err) { + return done(err); + } + + var pattern = 'unable to read ' + nonexistentFile; + expect(res, 'to satisfy', { + code: 1, + output: new RegExp(pattern, 'i') + }); + done(); + }, + spawnOpts + ); + }); +}); diff --git a/test/integration/options/retries.spec.js b/test/integration/options/retries.spec.js new file mode 100644 index 0000000000..2f2fdec869 --- /dev/null +++ b/test/integration/options/retries.spec.js @@ -0,0 +1,24 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--retries', function() { + var args = []; + + it('should retry test failures after a certain threshold', function(done) { + args = ['--retries', '3']; + var fixture = path.join('options', 'retries'); + runMochaJSON(fixture, args, function(err, res) { + if (err) { + return done(err); + } + expect(res, 'to have failed') + .and('not to have pending tests') + .and('not to have passed tests') + .and('to have retried test', 'should fail', 3); + done(); + }); + }); +}); diff --git a/test/integration/options/sort.spec.js b/test/integration/options/sort.spec.js new file mode 100644 index 0000000000..df9318762e --- /dev/null +++ b/test/integration/options/sort.spec.js @@ -0,0 +1,28 @@ +'use strict'; + +var path = require('path').posix; +var helpers = require('../helpers'); +var runMochaJSON = helpers.runMochaJSON; + +describe('--sort', function() { + var args = []; + + before(function() { + args = ['--sort']; + }); + + it('should sort tests in alphabetical order', function(done) { + var fixtures = path.join('options', 'sort*'); + runMochaJSON(fixtures, args, function(err, res) { + if (err) { + done(err); + return; + } + expect(res, 'to have passed test count', 2).and( + 'to have passed test order', + 'should be executed first' + ); + done(); + }); + }); +}); diff --git a/test/integration/options/watch.spec.js b/test/integration/options/watch.spec.js new file mode 100644 index 0000000000..1df65fa342 --- /dev/null +++ b/test/integration/options/watch.spec.js @@ -0,0 +1,56 @@ +'use strict'; + +var helpers = require('../helpers'); +var runMochaJSONRaw = helpers.runMochaJSONRaw; + +describe('--watch', function() { + var args = []; + + before(function() { + args = ['--watch']; + }); + + describe('when enabled', function() { + before(function() { + // Feature works but SIMULATING the signal (ctrl+c) via child process + // does not work due to lack of POSIX signal compliance on Windows. + if (process.platform === 'win32') { + this.skip(); + } + }); + + it('should show the cursor and signal correct exit code, when watch process is terminated', function(done) { + this.timeout(0); + this.slow(3000); + + var fixture = 'exit.fixture.js'; + var spawnOpts = {stdio: 'pipe'}; + var mocha = runMochaJSONRaw( + fixture, + args, + function postmortem(err, data) { + if (err) { + return done(err); + } + + var expectedCloseCursor = '\u001b[?25h'; + expect(data.output, 'to contain', expectedCloseCursor); + + function exitStatusBySignal(sig) { + return 128 + sig; + } + + var sigint = 2; + expect(data.code, 'to be', exitStatusBySignal(sigint)); + done(); + }, + spawnOpts + ); + + setTimeout(function() { + // Kill the child process + mocha.kill('SIGINT'); + }, 1000); + }); + }); +});