chore(eslint-plugin): add config validation script to CI (typescript-…

…eslint#649)

* chore(eslint-plugin): add config validation script to CI

* chore: correct docs

azure-pipelines.yml

@@ -32,9 +32,13 @@ jobs:
         displayName: 'Run linting'
 
       - script: |
-          yarn docs:check
+          yarn check:docs
         displayName: 'Validate documentation'
 
+      - script: |
+          yarn check:configs
+        displayName: 'Validate plugin configs'
+
       - script: |
           yarn test
         displayName: 'Run unit tests'

package.json

@@ -20,7 +20,8 @@
     "build": "lerna run build",
     "clean": "lerna clean && lerna run clean",
     "cz": "git-cz",
-    "docs:check": "lerna run docs:check",
+    "check:docs": "lerna run check:docs",
+    "check:configs": "lerna run check:configs",
     "generate-contributors": "yarn ts-node ./tools/generate-contributors.ts && yarn all-contributors generate",
     "format": "prettier --write \"./**/*.{ts,js,json,md}\"",
     "format-check": "prettier --list-different \"./**/*.{ts,js,json,md}\"",

packages/eslint-plugin/README.md

@@ -116,7 +116,6 @@ Then you should add `airbnb` (or `airbnb-base`) to your `extends` section of `.e
 
 ## Supported Rules
 
-<!-- Please run `npm run docs` to update this section -->
 <!-- begin rule list -->
 
 **Key**: :heavy_check_mark: = recommended, :wrench: = fixable, :thought_balloon: = requires type information
@@ -131,7 +130,7 @@ Then you should add `airbnb` (or `airbnb-base`) to your `extends` section of `.e
 | [`@typescript-eslint/ban-types`](./docs/rules/ban-types.md)                                               | Enforces that types will not to be used                                                                                                             | :heavy_check_mark: | :wrench: |                   |
 | [`@typescript-eslint/camelcase`](./docs/rules/camelcase.md)                                               | Enforce camelCase naming convention                                                                                                                 | :heavy_check_mark: |          |                   |
 | [`@typescript-eslint/class-name-casing`](./docs/rules/class-name-casing.md)                               | Require PascalCased class and interface names                                                                                                       | :heavy_check_mark: |          |                   |
-| [`@typescript-eslint/consistent-type-definitions`](./docs/rules/consistent-type-definitions.md)           | Consistent with type definition either `interface` or `type`                                                                                        | :heavy_check_mark: | :wrench: |                   |
+| [`@typescript-eslint/consistent-type-definitions`](./docs/rules/consistent-type-definitions.md)           | Consistent with type definition either `interface` or `type`                                                                                        |                    | :wrench: |                   |
 | [`@typescript-eslint/explicit-function-return-type`](./docs/rules/explicit-function-return-type.md)       | Require explicit return types on functions and class methods                                                                                        | :heavy_check_mark: |          |                   |
 | [`@typescript-eslint/explicit-member-accessibility`](./docs/rules/explicit-member-accessibility.md)       | Require explicit accessibility modifiers on class properties and methods                                                                            | :heavy_check_mark: |          |                   |
 | [`@typescript-eslint/func-call-spacing`](./docs/rules/func-call-spacing.md)                               | Require or disallow spacing between function identifiers and their invocations                                                                      |                    | :wrench: |                   |

packages/eslint-plugin/package.json

@@ -30,13 +30,12 @@
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
+    "check:docs": "ts-node --files ./tools/validate-docs/index.ts",
+    "check:configs": "ts-node --files ./tools/validate-configs/index.ts",
     "clean": "rimraf dist/",
-    "docs": "eslint-docs",
-    "docs:check": "ts-node --files ./tools/validate-docs/index.ts",
     "format": "prettier --write \"./**/*.{ts,js,json,md}\" --ignore-path ../../.prettierignore",
     "generate:configs": "ts-node --files tools/generate-configs.ts",
     "prebuild": "npm run clean",
-    "recommended:update": "ts-node tools/update-recommended.ts",
     "test": "jest --coverage",
     "typecheck": "tsc --noEmit"
   },

packages/eslint-plugin/src/configs/all.json

@@ -64,6 +64,7 @@
     "@typescript-eslint/restrict-plus-operands": "error",
     "semi": "off",
     "@typescript-eslint/semi": "error",
+    "@typescript-eslint/strict-boolean-expressions": "error",
     "@typescript-eslint/triple-slash-reference": "error",
     "@typescript-eslint/type-annotation-spacing": "error",
     "@typescript-eslint/unbound-method": "error",

packages/eslint-plugin/src/rules/consistent-type-definitions.ts

@@ -9,7 +9,7 @@ export default util.createRule({
       description:
         'Consistent with type definition either `interface` or `type`',
       category: 'Stylistic Issues',
-      recommended: 'error',
+      recommended: false,
     },
     messages: {
       interfaceOverType: 'Use an `interface` instead of a `type`',

packages/eslint-plugin/tools/validate-configs/checkConfigAll.ts

@@ -0,0 +1,35 @@
+import plugin from '../../src/index';
+import { logRule } from '../log';
+
+const prefix = '@typescript-eslint/';
+
+function checkConfigAll() {
+  const { rules } = plugin;
+
+  const all = plugin.configs.all.rules;
+  const allNames = new Set(Object.keys(all));
+
+  return Object.entries(rules).reduce((acc, [ruleName, rule]) => {
+    if (!rule.meta.deprecated) {
+      const prefixed = `${prefix}${ruleName}` as keyof typeof all;
+      if (allNames.has(prefixed)) {
+        if (all[prefixed] !== 'error') {
+          logRule(
+            false,
+            ruleName,
+            'incorrect setting compared to the rule meta.',
+          );
+          return true;
+        }
+      } else {
+        logRule(false, ruleName, 'missing in the config.');
+        return true;
+      }
+    }
+
+    logRule(true, ruleName);
+    return acc;
+  }, false);
+}
+
+export { checkConfigAll };

packages/eslint-plugin/tools/validate-configs/checkConfigRecommended.ts

@@ -0,0 +1,35 @@
+import plugin from '../../src/index';
+import { logRule } from '../log';
+
+const prefix = '@typescript-eslint/';
+
+function checkConfigRecommended() {
+  const { rules } = plugin;
+
+  const recommended = plugin.configs.recommended.rules;
+  const recommendedNames = new Set(Object.keys(recommended));
+
+  return Object.entries(rules).reduce((acc, [ruleName, rule]) => {
+    if (!rule.meta.deprecated && rule.meta.docs.recommended !== false) {
+      const prefixed = `${prefix}${ruleName}` as keyof typeof recommended;
+      if (recommendedNames.has(prefixed)) {
+        if (recommended[prefixed] !== rule.meta.docs.recommended) {
+          logRule(
+            false,
+            ruleName,
+            'incorrect setting compared to the rule meta.',
+          );
+          return true;
+        }
+      } else {
+        logRule(false, ruleName, 'missing in the config.');
+        return true;
+      }
+    }
+
+    logRule(true, ruleName);
+    return acc;
+  }, false);
+}
+
+export { checkConfigRecommended };

packages/eslint-plugin/tools/validate-configs/index.ts

@@ -0,0 +1,22 @@
+import chalk from 'chalk';
+import { checkConfigRecommended } from './checkConfigRecommended';
+import { checkConfigAll } from './checkConfigAll';
+
+let hasErrors = false;
+console.log(chalk.underline('Checking config "recommended"'));
+hasErrors = checkConfigRecommended() || hasErrors;
+
+console.log();
+console.log(chalk.underline('Checking config "all"'));
+hasErrors = checkConfigAll() || hasErrors;
+
+if (hasErrors) {
+  console.log('\n\n');
+  console.error(
+    chalk.bold.bgRed.white('There were errors found in the configs'),
+  );
+  console.error(`Please run ${chalk.inverse('yarn generate:configs')}`);
+  console.log('\n\n');
+  // eslint-disable-next-line no-process-exit
+  process.exit(1);
+}

packages/eslint-plugin/tools/validate-docs/check-for-rule-docs.ts

@@ -1,7 +1,7 @@
 import { TSESLint } from '@typescript-eslint/experimental-utils';
 import fs from 'fs';
 import path from 'path';
-import { logRule } from './log';
+import { logRule } from '../log';
 
 function checkForRuleDocs(
   rules: Record<string, Readonly<TSESLint.RuleModule<any, any, any>>>,

packages/eslint-plugin/tools/validate-docs/validate-table-rules.ts

@@ -3,7 +3,7 @@ import chalk from 'chalk';
 import fs from 'fs';
 import marked from 'marked';
 import path from 'path';
-import { logRule } from './log';
+import { logRule } from '../log';
 
 function validateTableRules(
   rules: Record<string, Readonly<TSESLint.RuleModule<any, any, any>>>,

packages/eslint-plugin/tools/validate-docs/validate-table-structure.ts

@@ -1,7 +1,7 @@
 import { TSESLint } from '@typescript-eslint/experimental-utils';
 import chalk from 'chalk';
 import marked from 'marked';
-import { logError } from './log';
+import { logError } from '../log';
 
 function validateTableStructure(
   rules: Record<string, Readonly<TSESLint.RuleModule<any, any, any>>>,