diff --git a/Makefile.js b/Makefile.js index 6cffb770c64..62b1d76669f 100644 --- a/Makefile.js +++ b/Makefile.js @@ -544,7 +544,7 @@ target.mocha = () => { echo("Running unit tests"); - lastReturn = exec(`${getBinFile("nyc")} -- ${MOCHA} -R progress -t ${MOCHA_TIMEOUT} -c ${TEST_FILES}`); + lastReturn = exec(`${getBinFile("nyc")} -- ${MOCHA} --forbid-only -R progress -t ${MOCHA_TIMEOUT} -c ${TEST_FILES}`); if (lastReturn.code !== 0) { errors++; } diff --git a/docs/developer-guide/nodejs-api.md b/docs/developer-guide/nodejs-api.md index 28b0fff9070..8d25c6e986d 100644 --- a/docs/developer-guide/nodejs-api.md +++ b/docs/developer-guide/nodejs-api.md @@ -1256,6 +1256,7 @@ A test case is an object with the following properties: * `code` (string, required): The source code that the rule should be run on * `options` (array, optional): The options passed to the rule. The rule severity should not be included in this list. * `filename` (string, optional): The filename for the given case (useful for rules that make assertions about filenames). +* `only` (boolean, optional): Run this case exclusively for debugging in supported test frameworks. In addition to the properties above, invalid test cases can also have the following properties: @@ -1355,10 +1356,13 @@ ruleTester.run("my-rule-for-no-foo", rule, { `RuleTester` depends on two functions to run tests: `describe` and `it`. These functions can come from various places: 1. If `RuleTester.describe` and `RuleTester.it` have been set to function values, `RuleTester` will use `RuleTester.describe` and `RuleTester.it` to run tests. You can use this to customize the behavior of `RuleTester` to match a test framework that you're using. -1. Otherwise, if `describe` and `it` are present as globals, `RuleTester` will use `global.describe` and `global.it` to run tests. This allows `RuleTester` to work when using frameworks like [Mocha](https://mochajs.org/) without any additional configuration. -1. Otherwise, `RuleTester#run` will simply execute all of the tests in sequence, and will throw an error if one of them fails. This means you can simply execute a test file that calls `RuleTester.run` using `node`, without needing a testing framework. -`RuleTester#run` calls the `describe` function with two arguments: a string describing the rule, and a callback function. The callback calls the `it` function with a string describing the test case, and a test function. The test function will return successfully if the test passes, and throw an error if the test fails. (Note that this is the standard behavior for test suites when using frameworks like [Mocha](https://mochajs.org/); this information is only relevant if you plan to customize `RuleTester.it` and `RuleTester.describe`.) + If `RuleTester.itOnly` has been set to a function value, `RuleTester` will call `RuleTester.itOnly` instead of `RuleTester.it` to run cases with `only: true`. If `RuleTester.itOnly` is not set but `RuleTester.it` has an `only` function property, `RuleTester` will fall back to `RuleTester.it.only`. + +2. Otherwise, if `describe` and `it` are present as globals, `RuleTester` will use `global.describe` and `global.it` to run tests and `global.it.only` to run cases with `only: true`. This allows `RuleTester` to work when using frameworks like [Mocha](https://mochajs.org/) without any additional configuration. +3. Otherwise, `RuleTester#run` will simply execute all of the tests in sequence, and will throw an error if one of them fails. This means you can simply execute a test file that calls `RuleTester.run` using `Node.js`, without needing a testing framework. + +`RuleTester#run` calls the `describe` function with two arguments: a string describing the rule, and a callback function. The callback calls the `it` function with a string describing the test case, and a test function. The test function will return successfully if the test passes, and throw an error if the test fails. The signature for `only` is the same as `it`. `RuleTester` calls either `it` or `only` for every case even when some cases have `only: true`, and the test framework is responsible for implementing test case exclusivity. (Note that this is the standard behavior for test suites when using frameworks like [Mocha](https://mochajs.org/); this information is only relevant if you plan to customize `RuleTester.describe`, `RuleTester.it`, or `RuleTester.itOnly`.) Example of customizing `RuleTester`: diff --git a/docs/developer-guide/unit-tests.md b/docs/developer-guide/unit-tests.md index 281c3967454..46d77b59076 100644 --- a/docs/developer-guide/unit-tests.md +++ b/docs/developer-guide/unit-tests.md @@ -10,11 +10,29 @@ This automatically starts Mocha and runs all tests in the `tests` directory. You ## Running Individual Tests -If you want to quickly run just one test, you can do so by running Mocha directly and passing in the filename. For example: +If you want to quickly run just one test file, you can do so by running Mocha directly and passing in the filename. For example: npm run test:cli tests/lib/rules/no-wrap-func.js -Running individual tests is useful when you're working on a specific bug and iterating on the solution. You should be sure to run `npm test` before submitting a pull request. +If you want to run just one or a subset of `RuleTester` test cases, add `only: true` to each test case or wrap the test case in `RuleTester.only(...)` to add it automatically: + +```js +ruleTester.run("my-rule", myRule, { + valid: [ + RuleTester.only("const valid = 42;"), + // Other valid cases + ], + invalid: [ + { + code: "const invalid = 42;", + only: true, + }, + // Other invalid cases + ] +}) +``` + +Running individual tests is useful when you're working on a specific bug and iterating on the solution. You should be sure to run `npm test` before submitting a pull request. `npm test` uses Mocha's `--forbid-only` option to prevent `only` tests from passing full test runs. ## More Control on Unit Testing diff --git a/lib/rule-tester/rule-tester.js b/lib/rule-tester/rule-tester.js index f177ae3cc17..cac81bc71d1 100644 --- a/lib/rule-tester/rule-tester.js +++ b/lib/rule-tester/rule-tester.js @@ -71,6 +71,7 @@ const espreePath = require.resolve("espree"); * @property {{ [name: string]: any }} [parserOptions] Options for the parser. * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. * @property {{ [name: string]: boolean }} [env] Environments for the test case. + * @property {boolean} [only] Run only this test case or the subset of test cases with this property. */ /** @@ -86,6 +87,7 @@ const espreePath = require.resolve("espree"); * @property {{ [name: string]: any }} [parserOptions] Options for the parser. * @property {{ [name: string]: "readonly" | "writable" | "off" }} [globals] The additional global variables. * @property {{ [name: string]: boolean }} [env] Environments for the test case. + * @property {boolean} [only] Run only this test case or the subset of test cases with this property. */ /** @@ -121,7 +123,8 @@ const RuleTesterParameters = [ "filename", "options", "errors", - "output" + "output", + "only" ]; /* @@ -282,6 +285,7 @@ function wrapParser(parser) { // default separators for testing const DESCRIBE = Symbol("describe"); const IT = Symbol("it"); +const IT_ONLY = Symbol("itOnly"); /** * This is `it` default handler if `it` don't exist. @@ -400,6 +404,46 @@ class RuleTester { this[IT] = value; } + /** + * Adds the `only` property to a test to run it in isolation. + * @param {string | ValidTestCase | InvalidTestCase} item A single test to run by itself. + * @returns {ValidTestCase | InvalidTestCase} The test with `only` set. + */ + static only(item) { + if (typeof item === "string") { + return { code: item, only: true }; + } + + return { ...item, only: true }; + } + + static get itOnly() { + if (typeof this[IT_ONLY] === "function") { + return this[IT_ONLY]; + } + if (typeof this[IT] === "function" && typeof this[IT].only === "function") { + return Function.bind.call(this[IT].only, this[IT]); + } + if (typeof it === "function" && typeof it.only === "function") { + return Function.bind.call(it.only, it); + } + + if (typeof this[DESCRIBE] === "function" || typeof this[IT] === "function") { + throw new Error( + "Set `RuleTester.itOnly` to use `only` with a custom test framework.\n" + + "See https://eslint.org/docs/developer-guide/nodejs-api#customizing-ruletester for more." + ); + } + if (typeof it === "function") { + throw new Error("The current test framework does not support exclusive tests with `only`."); + } + throw new Error("To use `only`, use RuleTester with a test framework that provides `it.only()` like Mocha."); + } + + static set itOnly(value) { + this[IT_ONLY] = value; + } + /** * Define a rule for one particular run of tests. * @param {string} name The name of the rule to define. @@ -891,23 +935,29 @@ class RuleTester { RuleTester.describe(ruleName, () => { RuleTester.describe("valid", () => { test.valid.forEach(valid => { - RuleTester.it(sanitize(typeof valid === "object" ? valid.code : valid), () => { - testValidTemplate(valid); - }); + RuleTester[valid.only ? "itOnly" : "it"]( + sanitize(typeof valid === "object" ? valid.code : valid), + () => { + testValidTemplate(valid); + } + ); }); }); RuleTester.describe("invalid", () => { test.invalid.forEach(invalid => { - RuleTester.it(sanitize(invalid.code), () => { - testInvalidTemplate(invalid); - }); + RuleTester[invalid.only ? "itOnly" : "it"]( + sanitize(invalid.code), + () => { + testInvalidTemplate(invalid); + } + ); }); }); }); } } -RuleTester[DESCRIBE] = RuleTester[IT] = null; +RuleTester[DESCRIBE] = RuleTester[IT] = RuleTester[IT_ONLY] = null; module.exports = RuleTester; diff --git a/tests/lib/rule-tester/rule-tester.js b/tests/lib/rule-tester/rule-tester.js index 3aaf6f91f3f..70647b18670 100644 --- a/tests/lib/rule-tester/rule-tester.js +++ b/tests/lib/rule-tester/rule-tester.js @@ -97,6 +97,266 @@ describe("RuleTester", () => { ruleTester = new RuleTester(); }); + describe("only", () => { + describe("`itOnly` accessor", () => { + describe("when `itOnly` is set", () => { + before(() => { + RuleTester.itOnly = sinon.spy(); + }); + after(() => { + RuleTester.itOnly = void 0; + }); + beforeEach(() => { + RuleTester.itOnly.resetHistory(); + ruleTester = new RuleTester(); + }); + + it("is called by exclusive tests", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(RuleTester.itOnly, "const notVar = 42;"); + }); + }); + + describe("when `it` is set and has an `only()` method", () => { + before(() => { + RuleTester.it.only = () => {}; + sinon.spy(RuleTester.it, "only"); + }); + after(() => { + RuleTester.it.only = void 0; + }); + beforeEach(() => { + RuleTester.it.only.resetHistory(); + ruleTester = new RuleTester(); + }); + + it("is called by tests with `only` set", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(RuleTester.it.only, "const notVar = 42;"); + }); + }); + + describe("when global `it` is a function that has an `only()` method", () => { + let originalGlobalItOnly; + + before(() => { + + /* + * We run tests with `--forbid-only`, so we have to override + * `it.only` to prevent the real one from being called. + */ + originalGlobalItOnly = it.only; + it.only = () => {}; + sinon.spy(it, "only"); + }); + after(() => { + it.only = originalGlobalItOnly; + }); + beforeEach(() => { + it.only.resetHistory(); + ruleTester = new RuleTester(); + }); + + it("is called by tests with `only` set", () => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + + sinon.assert.calledWith(it.only, "const notVar = 42;"); + }); + }); + + describe("when `describe` and `it` are overridden without `itOnly`", () => { + let originalGlobalItOnly; + + before(() => { + + /* + * These tests override `describe` and `it` already, so we + * don't need to override them here. We do, however, need to + * remove `only` from the global `it` to prevent it from + * being used instead. + */ + originalGlobalItOnly = it.only; + it.only = void 0; + }); + after(() => { + it.only = originalGlobalItOnly; + }); + beforeEach(() => { + ruleTester = new RuleTester(); + }); + + it("throws an error recommending overriding `itOnly`", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + }, "Set `RuleTester.itOnly` to use `only` with a custom test framework."); + }); + }); + + describe("when global `it` is a function that does not have an `only()` method", () => { + let originalGlobalIt; + let originalRuleTesterDescribe; + let originalRuleTesterIt; + + before(() => { + originalGlobalIt = global.it; + + // eslint-disable-next-line no-global-assign + it = () => {}; + + /* + * These tests override `describe` and `it`, so we need to + * un-override them here so they won't interfere. + */ + originalRuleTesterDescribe = RuleTester.describe; + RuleTester.describe = void 0; + originalRuleTesterIt = RuleTester.it; + RuleTester.it = void 0; + }); + after(() => { + + // eslint-disable-next-line no-global-assign + it = originalGlobalIt; + RuleTester.describe = originalRuleTesterDescribe; + RuleTester.it = originalRuleTesterIt; + }); + beforeEach(() => { + ruleTester = new RuleTester(); + }); + + it("throws an error explaining that the current test framework does not support `only`", () => { + assert.throws(() => { + ruleTester.run("no-var", require("../../fixtures/testers/rule-tester/no-var"), { + valid: [{ + code: "const notVar = 42;", + only: true + }], + invalid: [] + }); + }, "The current test framework does not support exclusive tests with `only`."); + }); + }); + }); + + describe("test cases", () => { + const ruleName = "no-var"; + const rule = require("../../fixtures/testers/rule-tester/no-var"); + + let originalRuleTesterIt; + let spyRuleTesterIt; + let originalRuleTesterItOnly; + let spyRuleTesterItOnly; + + before(() => { + originalRuleTesterIt = RuleTester.it; + spyRuleTesterIt = sinon.spy(); + RuleTester.it = spyRuleTesterIt; + originalRuleTesterItOnly = RuleTester.itOnly; + spyRuleTesterItOnly = sinon.spy(); + RuleTester.itOnly = spyRuleTesterItOnly; + }); + after(() => { + RuleTester.it = originalRuleTesterIt; + RuleTester.itOnly = originalRuleTesterItOnly; + }); + beforeEach(() => { + spyRuleTesterIt.resetHistory(); + spyRuleTesterItOnly.resetHistory(); + ruleTester = new RuleTester(); + }); + + it("isn't called for normal tests", () => { + ruleTester.run(ruleName, rule, { + valid: ["const notVar = 42;"], + invalid: [] + }); + sinon.assert.calledWith(spyRuleTesterIt, "const notVar = 42;"); + sinon.assert.notCalled(spyRuleTesterItOnly); + }); + + it("calls it or itOnly for every test case", () => { + + /* + * `RuleTester` doesn't implement test case exclusivity itself. + * Setting `only: true` just causes `RuleTester` to call + * whatever `only()` function is provided by the test framework + * instead of the regular `it()` function. + */ + + ruleTester.run(ruleName, rule, { + valid: [ + "const valid = 42;", + { + code: "const onlyValid = 42;", + only: true + } + ], + invalid: [ + { + code: "var invalid = 42;", + errors: [/^Bad var/u] + }, + { + code: "var onlyInvalid = 42;", + errors: [/^Bad var/u], + only: true + } + ] + }); + + sinon.assert.calledWith(spyRuleTesterIt, "const valid = 42;"); + sinon.assert.calledWith(spyRuleTesterItOnly, "const onlyValid = 42;"); + sinon.assert.calledWith(spyRuleTesterIt, "var invalid = 42;"); + sinon.assert.calledWith(spyRuleTesterItOnly, "var onlyInvalid = 42;"); + }); + }); + + describe("static helper wrapper", () => { + it("adds `only` to string test cases", () => { + const test = RuleTester.only("const valid = 42;"); + + assert.deepStrictEqual(test, { + code: "const valid = 42;", + only: true + }); + }); + + it("adds `only` to object test cases", () => { + const test = RuleTester.only({ code: "const valid = 42;" }); + + assert.deepStrictEqual(test, { + code: "const valid = 42;", + only: true + }); + }); + }); + }); + it("should not throw an error when everything passes", () => { ruleTester.run("no-eval", require("../../fixtures/testers/rule-tester/no-eval"), { valid: [