feat: add new package rule-tester

.cspell.json

@@ -1,6 +1,14 @@
 {
-  "version": "0.1",
+  "version": "0.2",
   "language": "en",
+  "enableFiletypes": [
+    "markdown",
+    "mdx",
+    "typescript",
+    "typescriptreact",
+    "javascript",
+    "javascriptreact"
+  ],
   "ignorePaths": [
     ".cspell.json",
     ".github/workflows/**",

.eslintignore

@@ -12,3 +12,6 @@ packages/types/src/generated/**/*.ts
 
 # Playground types downloaded from the web
 packages/website/src/vendor
+
+# see the file header in eslint-base.test.js for more info
+packages/rule-tester/tests/eslint-base

.github/renovate.json5

@@ -1,6 +1,8 @@
 {
   enabledManagers: ['github-actions', 'npm'],
   ignoreDeps: [
+    // AJV is out-of-date, but it's intentionally synced with ESLint - https://github.com/eslint/eslint/blob/ad9dd6a933fd098a0d99c6a9aa059850535c23ee/package.json#L70
+    'ajv',
     // globby is ESM so we can't go any higher right now
     'globby',
     // this dep now uses package.json exports - we will be removing it next major

.prettierignore

@@ -18,3 +18,6 @@ CHANGELOG.md
 
 packages/website/.docusaurus
 packages/website/build
+
+# see the file header in eslint-base.test.js for more info
+packages/rule-tester/tests/eslint-base

.vscode/launch.json

@@ -105,6 +105,42 @@
         "${workspaceFolder}/packages/scope-manager/dist/index.js",
       ],
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Run currently opened rule-tester test",
+      "cwd": "${workspaceFolder}/packages/rule-tester/",
+      "program": "${workspaceFolder}/node_modules/jest/bin/jest.js",
+      "args": [
+        "--runInBand",
+        "--no-cache",
+        "--no-coverage",
+        "${fileBasename}"
+      ],
+      "sourceMaps": true,
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "skipFiles": [
+        "${workspaceFolder}/packages/utils/src/index.ts",
+        "${workspaceFolder}/packages/utils/dist/index.js",
+        "${workspaceFolder}/packages/utils/src/ts-estree.ts",
+        "${workspaceFolder}/packages/utils/dist/ts-estree.js",
+        "${workspaceFolder}/packages/type-utils/src/ts-estree.ts",
+        "${workspaceFolder}/packages/type-utils/dist/ts-estree.js",
+        "${workspaceFolder}/packages/parser/src/index.ts",
+        "${workspaceFolder}/packages/parser/dist/index.js",
+        "${workspaceFolder}/packages/rule-tester/src/index.ts",
+        "${workspaceFolder}/packages/rule-tester/dist/index.js",
+        "${workspaceFolder}/packages/typescript-estree/src/index.ts",
+        "${workspaceFolder}/packages/typescript-estree/dist/index.js",
+        "${workspaceFolder}/packages/types/src/index.ts",
+        "${workspaceFolder}/packages/types/dist/index.js",
+        "${workspaceFolder}/packages/visitor-keys/src/index.ts",
+        "${workspaceFolder}/packages/visitor-keys/dist/index.js",
+        "${workspaceFolder}/packages/scope-manager/dist/index.js",
+        "${workspaceFolder}/packages/scope-manager/dist/index.js",
+      ],
+    },
     {
       "type": "node",
       "request": "launch",

docs/Architecture.mdx

@@ -12,6 +12,7 @@ They are:
 - [`@typescript-eslint/eslint-plugin`](./architecture/ESLint_Plugin.mdx): An ESLint plugin which provides lint rules for TypeScript codebases.
 - [`@typescript-eslint/eslint-plugin-tslint`](./architecture/ESLint_Plugin_TSLint.mdx): ESLint plugin that allows running TSLint rules within ESLint to help you migrate from TSLint to ESLint.
 - [`@typescript-eslint/parser`](./architecture/Parser.mdx): An ESLint parser which allows for ESLint to lint TypeScript source code.
+- [`@typescript-eslint/rule-tester`](./architecture/Rule_Tester.mdx): A utility for testing ESLint rules.
 - [`@typescript-eslint/scope-manager`](./architecture/Scope_Manager.mdx): A fork of [`eslint-scope`](https://github.com/eslint/eslint-scope), enhanced to support TypeScript functionality.
-- [`@typescript-eslint/typescript-estree`](./architecture/TypeScript-ESTree.mdx): The underlying code used by [`@typescript-eslint/parser`](./architecture/Parser.mdx) that converts TypeScript source code into an <a href="https://github.com/estree/estree">ESTree</a>-compatible form.
+- [`@typescript-eslint/typescript-estree`](./architecture/TypeScript_ESTree.mdx): The underlying code used by [`@typescript-eslint/parser`](./architecture/Parser.mdx) that converts TypeScript source code into an <a href="https://github.com/estree/estree">ESTree</a>-compatible form.
 - [`@typescript-eslint/utils`](./architecture/Utils.mdx): Utilities for working with TypeScript + ESLint together.

docs/Custom_Rules.mdx

@@ -274,18 +274,20 @@ This can be necessary for TypeScript APIs not wrapped by the parser services.
 
 ## Testing
 
-`@typescript-eslint/utils` exports a `RuleTester` with a similar API to the built-in [ESLint `RuleTester`](https://eslint.org/docs/developer-guide/nodejs-api#ruletester).
+`@typescript-eslint/rule-tester` exports a `RuleTester` with a similar API to the built-in ESLint `RuleTester`.
 It should be provided with the same `parser` and `parserOptions` you would use in your ESLint configuration.
 
+Below is a quick-start guide. For more in-depth docs and examples [see the `@typescript-eslint/rule-tester` package documentation](./architecture/Rule_Tester.mdx).
+
 ### Testing Untyped Rules
 
 For rules that don't need type information, passing just the `parser` will do:
 
 ```ts
-import { ESLintUtils } from '@typescript-eslint/utils';
+import { RuleTester } from '@typescript-eslint/rule-tester';
 import rule from './my-rule';
 
-const ruleTester = new ESLintUtils.RuleTester({
+const ruleTester = new RuleTester({
   parser: '@typescript-eslint/parser',
 });
 
@@ -305,10 +307,10 @@ For rules that do need type information, `parserOptions` must be passed in as we
 Tests must have at least an absolute `tsconfigRootDir` path provided as well as a relative `project` path from that directory:
 
 ```ts
-import { ESLintUtils } from '@typescript-eslint/utils';
+import { RuleTester } from '@typescript-eslint/rule-tester';
 import rule from './my-typed-rule';
 
-const ruleTester = new ESLintUtils.RuleTester({
+const ruleTester = new RuleTester({
   parser: '@typescript-eslint/parser',
   parserOptions: {
     project: './tsconfig.json',
@@ -327,11 +329,11 @@ ruleTester.run('my-typed-rule', rule, {
 ```
 
 :::note
-For now, `ESLintUtils.RuleTester` requires the following physical files be present on disk for typed rules:
+For now, `RuleTester` requires the following physical files be present on disk for typed rules:
 
 - `tsconfig.json`: tsconfig used as the test "project"
 - One of the following two files:
   - `file.ts`: blank test file used for normal TS tests
-  - `file.tsx`: blank test file used for tests with `parserOptions: { ecmaFeatures: { jsx: true } }`
+  - `react.tsx`: blank test file used for tests with `parserOptions: { ecmaFeatures: { jsx: true } }`
 
 :::

docs/architecture/Rule_Tester.mdx

@@ -0,0 +1,218 @@
+---
+id: rule-tester
+sidebar_label: rule-tester
+---
+
+import CodeBlock from '@theme/CodeBlock';
+
+# `@typescript-eslint/rule-tester`
+
+> A utility for testing ESLint rules
+
+This is a fork of ESLint's built-in `RuleTester` to provide some better types and additional features for testing TypeScript rules.
+
+## Usage
+
+For non-type-aware rules you can test them as follows:
+
+```ts
+import { RuleTester } from '@typescript-eslint/rule-tester';
+import rule from '../src/rules/my-rule.ts';
+
+const ruleTester = new RuleTester({
+  parser: '@typescript-eslint/parser',
+});
+
+ruleTester.run('my-rule', rule, {
+  valid: [
+    // valid tests can be a raw string,
+    'const x = 1;',
+    // or they can be an object
+    {
+      code: 'const y = 2;',
+      options: [{ ruleOption: true }],
+    },
+
+    // you can enable JSX parsing by passing parserOptions.ecmaFeatures.jsx = true
+    {
+      code: 'const z = <div />;',
+      parserOptions: {
+        ecmaFeatures: {
+          jsx: true,
+        },
+      },
+    },
+  ],
+  invalid: [
+    // invalid tests must always be an object
+    {
+      code: 'const a = 1;',
+      // invalid tests must always specify the expected errors
+      errors: [
+        {
+          messageId: 'ruleMessage',
+          // If applicable - it's recommended that you also assert the data in
+          // addition to the messageId so that you can ensure the correct message
+          // is generated
+          data: {
+            placeholder1: 'a',
+          },
+        },
+      ],
+    },
+
+    // fixers can be tested using the output parameter
+    {
+      code: 'const b = 1;',
+      output: 'const c = 1;',
+      errors: [
+        /* ... */
+      ],
+    },
+    // passing `output = null` will enforce the code is NOT changed
+    {
+      code: 'const c = 1;',
+      output: null,
+      errors: [
+        /* ... */
+      ],
+    },
+
+    // suggestions can be tested via errors
+    {
+      code: 'const d = 1;',
+      output: null,
+      errors: [
+        {
+          messageId: 'suggestionError',
+          suggestions: [
+            {
+              messageId: 'suggestionOne',
+              output: 'const e = 1;',
+            },
+          ],
+        },
+      ],
+    },
+    // passing `suggestions = null` will enforce there are NO suggestions
+    {
+      code: 'const d = 1;',
+      output: null,
+      errors: [
+        {
+          messageId: 'noSuggestionError',
+          suggestions: null,
+        },
+      ],
+    },
+  ],
+});
+```
+
+### Type-Aware Testing
+
+Type-aware rules can be tested in almost exactly the same way, except you need to create some files on disk.
+We require files on disk due to a limitation with TypeScript in that it requires physical files on disk to initialize the project.
+We suggest creating a `fixture` folder nearby that contains three files:
+
+1. `file.ts` - this should be an empty file.
+2. `react.tsx` - this should be an empty file.
+3. `tsconfig.json` - this should be the config to use for your test, for example:
+   ```json
+   {
+     "compilerOptions": {
+       "strict": true
+     },
+     "include": ["file.ts", "react.tsx"]
+   }
+   ```
+
+:::caution
+It's important to note that both `file.ts` and `react.tsx` must both be empty files!
+The rule tester will automatically use the string content from your tests - the empty files are just there for initialization.
+:::
+
+You can then test your rule by providing the type-aware config:
+
+```ts
+const ruleTester = new RuleTester({
+  parser: '@typescript-eslint/parser',
+  // Added lines start
+  parserOptions: {
+    tsconfigRootDir: './path/to/your/folder/fixture',
+    project: './tsconfig.json',
+  },
+  // Added lines end
+});
+```
+
+With that config the parser will automatically run in type-aware mode and you can write tests just like before.
+
+### Test Dependency Constraints
+
+Sometimes it's desirable to test your rule against multiple versions of a dependency to ensure backwards and forwards compatibility.
+With backwards-compatibility testing there comes a complication in that some tests may not be compatible with an older version of a dependency.
+For example - if you're testing against an older version of TypeScript, certain features might cause a parser error!
+
+import DependencyConstraint from '!!raw-loader!../../packages/rule-tester/src/types/DependencyConstraint.ts';
+
+<CodeBlock language="ts">{DependencyConstraint}</CodeBlock>
+
+The `RuleTester` allows you to apply dependency constraints at either an individual test or constructor level.
+
+```ts
+const ruleTester = new RuleTester({
+  parser: '@typescript-eslint/parser',
+  // Added lines start
+  dependencyConstraints: {
+    // none of the tests will run unless `my-dependency` matches the semver range `>=1.2.3`
+    'my-dependency': '1.2.3',
+    // you can also provide granular semver ranges
+    'my-granular-dep': {
+      // none of the tests will run unless `my-granular-dep` matches the semver range `~3.2.1`
+      range: '~3.2.1',
+    },
+  },
+  // Added lines end
+});
+
+ruleTester.run('my-rule', rule, {
+  valid: [
+    {
+      code: 'const y = 2;',
+      // Added lines start
+      dependencyConstraints: {
+        // this test won't run unless BOTH dependencies match the given ranges
+        first: '1.2.3',
+        second: '3.2.1',
+      },
+      // Added lines end
+    },
+  ],
+  invalid: [
+    /* ... */
+  ],
+});
+```
+
+All dependencies provided in the `dependencyConstraints` object must match their given ranges in order for a test to not be skipped.
+
+## Options
+
+### `RuleTester` constructor options
+
+import RuleTesterConfig from '!!raw-loader!../../packages/rule-tester/src/types/RuleTesterConfig.ts';
+
+<CodeBlock language="ts">{RuleTesterConfig}</CodeBlock>
+
+### Valid test case options
+
+import ValidTestCase from '!!raw-loader!../../packages/rule-tester/src/types/ValidTestCase.ts';
+
+<CodeBlock language="ts">{ValidTestCase}</CodeBlock>
+
+### Invalid test case options
+
+import InvalidTestCase from '!!raw-loader!../../packages/rule-tester/src/types/InvalidTestCase.ts';
+
+<CodeBlock language="ts">{InvalidTestCase}</CodeBlock>

packages/eslint-plugin-tslint/README.md

@@ -8,3 +8,5 @@
 👉 See **https://typescript-eslint.io/architecture/eslint-plugin-tslint** for documentation on this package.
 
 > See https://typescript-eslint.io for general documentation on typescript-eslint, the tooling that allows you to run ESLint and Prettier on TypeScript code.
+
+<!-- Local path for docs: docs/architecture/ESLint_Plugin_TSLint.mdx -->

packages/eslint-plugin/README.md

@@ -8,3 +8,5 @@ An ESLint plugin which provides lint rules for TypeScript codebases.
 👉 See **https://typescript-eslint.io/getting-started** for our Getting Started docs.
 
 > See https://typescript-eslint.io for general documentation on typescript-eslint, the tooling that allows you to run ESLint and Prettier on TypeScript code.
+
+<!-- Local path for docs: docs/architecture/ESLint_Plugin.mdx -->

packages/parser/README.md

@@ -8,3 +8,5 @@
 👉 See **https://typescript-eslint.io/architecture/parser** for documentation on this package.
 
 > See https://typescript-eslint.io for general documentation on typescript-eslint, the tooling that allows you to run ESLint and Prettier on TypeScript code.
+
+<!-- Local path for docs: docs/architecture/Parser.mdx -->