chore: implement a test runner on top of mocha (#8866)

* chore: implement a test runner on top of mocha

This PR implements a test runner on top of mocha
that performs multiple mocha runs as defined in
TestSuites.json and compares the outcome of the runs
against TestExpectations.json. This allows us to
remove most of helpers from mocha-utils and be more
flexible when defining the test configurations.

.github/workflows/ci.yml

@@ -173,26 +173,12 @@ jobs:
         run: npm run build:dev
       - name: Test types
         run: npm run test:types
-      # On Linux we run all Chrome tests without retries and Firefox tests with retries.
-      - name: Run all Chrome tests (only on Linux)
+      - name: Run all tests with xvfb
         if: ${{ matrix.spec.name == 'Linux' }}
-        run: xvfb-run --auto-servernum npm run test:chrome
-      - name: Run all Firefox tests (only on Linux)
-        if: ${{ matrix.spec.name == 'Linux' }}
-        uses: nick-invision/retry@v2
-        with:
-          command: xvfb-run --auto-servernum npm run test:firefox
-          timeout_minutes: 20
-          max_attempts: 3
-      # On other platforms we only run chrome:headless tests and continue on Firefox errors.
-      - name: Test Chrome Headless (not on Linux)
-        id: test-chrome
-        if: ${{ matrix.spec.name != 'Linux' }}
-        run: npm run test:chrome:headless
-      - name: Run all Firefox tests (not on Linux)
+        run: xvfb-run --auto-servernum npm run test
+      - name: Run all tests without xvfb
         if: ${{ matrix.spec.name != 'Linux' }}
-        continue-on-error: true
-        run: npm run test:firefox
+        run: npm run test
       - name: Test bundling and installation
         if: ${{ matrix.spec.name == 'Linux' }}
         run: |

.mocharc.cjs

@@ -22,6 +22,6 @@ module.exports = {
   exit: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   parallel: !!process.env.PARALLEL,
-  timeout: 25 * 1000,
+  timeout: 25_000,
   reporter: process.env.CI ? 'spec' : 'dot',
 };

package.json

@@ -27,14 +27,14 @@
     "node": ">=14.1.0"
   },
   "scripts": {
-    "test": "c8 --check-coverage --lines 93 run-s test:chrome:* test:firefox",
+    "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 c8 --check-coverage --lines 93 node utils/mochaRunner/lib/main.js",
     "test:types": "tsd",
     "test:install": "scripts/test-install.sh",
-    "test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha",
+    "test:firefox": "npm run test -- --test-suite firefox-headless",
     "test:chrome": "run-s test:chrome:*",
-    "test:chrome:headless": "cross-env HEADLESS=true PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha",
-    "test:chrome:headless-chrome": "cross-env HEADLESS=chrome PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha",
-    "test:chrome:headful": "cross-env HEADLESS=false PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha",
+    "test:chrome:headless": "npm run test -- --test-suite chrome-headless",
+    "test:chrome:headless-chrome": "npm run test -- --test-suite chrome-new-headless",
+    "test:chrome:headful": "npm run test -- --test-suite chrome-headful",
     "prepublishOnly": "npm run build",
     "prepare": "node typescript-if-required.js && husky install",
     "lint": "run-s lint:prettier lint:eslint",
@@ -139,6 +139,7 @@
     "text-diff": "1.0.1",
     "tsd": "0.22.0",
     "tsx": "3.8.2",
-    "typescript": "4.7.4"
+    "typescript": "4.7.4",
+    "zod": "3.18.0"
   }
 }