test: rework the integration tests to not use docker (#4078)

.eslintrc.js

@@ -22,8 +22,8 @@ module.exports = {
     sourceType: 'module',
     project: [
       './tsconfig.eslint.json',
-      './tests/integration/utils/jsconfig.json',
       './packages/*/tsconfig.json',
+      './tests/integration/tsconfig.json',
     ],
     allowAutomaticSingleRunInference: true,
     tsconfigRootDir: __dirname,
@@ -175,12 +175,16 @@ module.exports = {
         'packages/*/tests/**/spec.ts',
         'packages/*/tests/**/test.ts',
         'packages/parser/tests/**/*.ts',
+        'tests/integration/**/*.test.ts',
+        'tests/integration/integration-test-base.ts',
+        'tests/integration/pack-packages.ts',
       ],
       env: {
         'jest/globals': true,
       },
       rules: {
         '@typescript-eslint/no-unsafe-assignment': 'off',
+        '@typescript-eslint/no-unsafe-call': 'off',
         '@typescript-eslint/no-unsafe-member-access': 'off',
         '@typescript-eslint/no-unsafe-return': 'off',
         'eslint-plugin/no-identical-tests': 'error',

.github/workflows/ci.yml

@@ -343,46 +343,3 @@ jobs:
         run: npx lerna publish --loglevel=verbose --canary --exact --force-publish --yes
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-
-  publish_canary_version_v5:
-    name: Publish the latest code as a canary version
-    runs-on: ubuntu-latest
-    needs: [typecheck, test_on_primary_node_version, unit_tests_on_other_node_versions, linting_and_style, integration_tests]
-    if: github.repository == 'typescript-eslint/typescript-eslint' && github.ref == 'refs/heads/v5'
-    steps:
-      - uses: actions/checkout@v2
-      # Fetch all history for all tags and branches in this job because lerna needs it
-      - run: |
-          git fetch --prune --unshallow
-
-      - name: Use Node.js ${{ env.PRIMARY_NODE_VERSION }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ env.PRIMARY_NODE_VERSION }}
-          registry-url: https://registry.npmjs.org/
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-
-      - uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Install dependencies
-        run: |
-          yarn --ignore-engines --frozen-lockfile --ignore-scripts
-          yarn check-clean-workspace-after-install
-
-      - name: Build
-        run: |
-          yarn build
-
-      - name: Publish all packages to npm
-        run: npx lerna publish premajor --loglevel=verbose --canary --exact --force-publish --yes --dist-tag rc-v5
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package.json

@@ -35,7 +35,7 @@
     "pre-commit": "yarn lint-staged",
     "pre-push": "yarn check-format",
     "test": "nx run-many --target=test --all --parallel",
-    "test-integration": "./tests/integration/run-all-tests.sh",
+    "test-integration": "yarn jest -c ./tests/integration/jest.config.js",
     "test-kill-integration-containers": "docker-compose -f tests/integration/docker-compose.yml down -v --rmi local",
     "typecheck": "nx run-many --target=typecheck --all --parallel"
   },
@@ -90,6 +90,7 @@
     "@types/jest-specific-snapshot": "^0.5.5",
     "@types/lodash": "^4.14.176",
     "@types/marked": "^3.0.2",
+    "@types/ncp": "^2.0.5",
     "@types/node": "^16.11.4",
     "@types/prettier": "^2.3.2",
     "@types/rimraf": "^3.0.2",
@@ -114,10 +115,12 @@
     "lint-staged": "^11.1.2",
     "make-dir": "^3.1.0",
     "markdownlint-cli": "^0.29.0",
+    "ncp": "^2.0.0",
     "node-fetch": "^3.0.0",
     "prettier": "2.4.1",
     "pretty-format": "^27.3.1",
     "rimraf": "^3.0.2",
+    "tmp": "^0.2.1",
     "ts-jest": "^27.0.5",
     "ts-node": "^10.4.0",
     "tslint": "^6.1.3",

packages/typescript-estree/package.json

@@ -63,7 +63,7 @@
     "glob": "*",
     "jest-specific-snapshot": "*",
     "make-dir": "*",
-    "tmp": "^0.2.1",
+    "tmp": "*",
     "typescript": "*"
   },
   "peerDependenciesMeta": {

packages/typescript-estree/tests/lib/convert.test.ts

@@ -1,5 +1,5 @@
 // deeplyCopy is private internal
-/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import * as ts from 'typescript';
 import type { TSNode } from '../../src';
 import { Converter } from '../../src/convert';

tests/integration/README.md

@@ -2,19 +2,30 @@
 
 We have a set of integration tests defined in this project to help ensure we don't inadvertently break downstream packages that depend on us.
 
-These tests are setup to run within docker containers to ensure that each test is completely isolated; we don't want them to affect our local environment, and similarly we don't want them to be effected by our local environment.
+These tests are setup to run within temporary folders to ensure that each test is isolated from the project.
 
 ## Adding a new integration test
 
-1. [Install docker for your platform](https://docs.docker.com/v17.09/engine/installation/#supported-platforms).
 1. Add a new folder in `/tests/integration/fixtures`
-1. Add a `.eslintrc.yml`, and a `tsconfig.json` to your folder, with the config required.
+1. Add a `package.json` to your folder.
+1. List the required dependencies under `devDependencies`.
+   - Use `latest` for the dependency to ensure we are testing against the newest versions of the package.
+   - If you have no dependencies, just add `"devDependencies": {}`.
+1. Add a `.eslintrc.js`, and a `tsconfig.json` to your folder, with all of the config required.
 1. Create the necessary files to test the integration.
-1. Copy+paste the `Dockerfile` from an existing fixture (they are all the same).
-1. Copy+paste the `test.sh` from an existing fixture, and adjust the `eslint` command as required.
-1. Add a new entry to `docker-compose.yml` by copy+pasting an existing section, and changing the name to match your new folder.
-1. Add a new entry to `run-all-tests.sh` by copy+pasting an existing command, and changing the name to match your new folder.
-1. Run your integration test by running the single command you copied in the previous step.
-   - If your test finishes successfully, a `test.js.snap` will be created.
+   - Your test should have a lint error in it in an appropriate location.
+     This is so that we can be certain the setup actually works correctly.
+1. Add a test to `/tests/integration/tests` named the same as your folder.
+1. Paste the following content into your test:
 
-If you run your test and see the test fail with `Cannot find module './lint-output.json' from 'test.js'`, this means that ESLint errored whilst attempting to run the lint command.
+   ```ts
+   import { integrationTest } from '../integration-test-base';
+
+   integrationTest(
+     __filename,
+     '*.ts' /* UPDATE THIS TO THE EXTENSION(s) TO LINT */,
+   );
+   ```
+
+1. Run your integration test with `yarn test-integration ./tests/integration/tests/your-file.test.ts`
+   - This will generate your snapshot output for the lint run which is a JSON representation of your ESLint run.

tests/integration/fixtures/eslint-v6/.eslintrc.js

@@ -1,11 +1,7 @@
 module.exports = {
   root: true,
-  // Local version of @typescript-eslint/parser
   parser: '@typescript-eslint/parser',
-  plugins: [
-    // Local version of @typescript-eslint/eslint-plugin
-    '@typescript-eslint',
-  ],
+  plugins: ['@typescript-eslint'],
   rules: {
     '@typescript-eslint/no-explicit-any': 'error',
   },

tests/integration/fixtures/eslint-v6/package.json

@@ -0,0 +1,5 @@
+{
+  "devDependencies": {
+    "eslint": "6.0.0"
+  }
+}