Update test config (#43661)

Updates our test config to take advantage of more concurrency and also updates to leverage the playwright docker image to reduce flakes from actions/setup-node stalling or playwright dependencies stalling on `apt install`. This reduces our test times from upwards of 30 minutes down to back around 15 minutes.
vercel · Dec 4, 2022 · 90f022a · 90f022a
1 parent 2fcc1e2
commit 90f022a
Show file tree

Hide file tree

Showing 9 changed files with 163 additions and 120 deletions.
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
@@ -261,15 +261,8 @@ jobs:
       fail-fast: false
       matrix:
         node: [16, 18]
-        group: [1, 2]
+        group: [1, 2, 3, 4]
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-        with:
-          node-version: ${{ matrix.node }}
-          check-latest: true
-
       - run: echo ${{needs.build.outputs.docsChange}}
 
       # https://github.com/actions/virtual-environments/issues/1187
@@ -289,16 +282,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: npm i -g pnpm@${PNPM_VERSION}
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      - run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
-        timeout-minutes: 10
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      # - run: npx @replayio/playwright install chromium
-      #   if: ${{needs.build.outputs.docsChange == 'nope'}}
-      - run: node run-tests.js --type development --timings -g ${{ matrix.group }}/2
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type development --timings -g ${{ matrix.group }}/4 >> /proc/1/fd/1"
         name: Run test/development
         if: ${{needs.build.outputs.docsChange == 'nope'}}
         # env:
@@ -338,15 +322,8 @@ jobs:
       fail-fast: false
       matrix:
         node: [16, 18]
-        group: [1, 2, 3]
+        group: [1, 2, 3, 4, 5, 6, 7]
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-        with:
-          node-version: ${{ matrix.node }}
-          check-latest: true
-
       - run: echo ${{needs.build.outputs.docsChange}}
 
       # https://github.com/actions/virtual-environments/issues/1187
@@ -366,16 +343,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: npm i -g pnpm@${PNPM_VERSION}
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      - run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
-        timeout-minutes: 10
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      # - run: npx @replayio/playwright install chromium
-      #   if: ${{needs.build.outputs.docsChange == 'nope'}}
-      - run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/7 >> /proc/1/fd/1"
         name: Run test/e2e (dev)
         if: ${{needs.build.outputs.docsChange == 'nope'}}
         env:
@@ -415,15 +383,8 @@ jobs:
       fail-fast: false
       matrix:
         node: [16, 18]
-        group: [1, 2]
+        group: [1, 2, 3]
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-        with:
-          node-version: ${{ matrix.node }}
-          check-latest: true
-
       - run: echo ${{needs.build.outputs.docsChange}}
 
       # https://github.com/actions/virtual-environments/issues/1187
@@ -443,16 +404,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: npm i -g pnpm@${PNPM_VERSION}
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      - run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
-        timeout-minutes: 10
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      # - run: npx @replayio/playwright install chromium
-      #   if: ${{needs.build.outputs.docsChange == 'nope'}}
-      - run: node run-tests.js --type production --timings -g ${{ matrix.group }}/2
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type production --timings -g ${{ matrix.group }}/3 >> /proc/1/fd/1"
         name: Run test/production
         if: ${{needs.build.outputs.docsChange == 'nope'}}
         # env:
@@ -482,15 +434,8 @@ jobs:
       fail-fast: false
       matrix:
         node: [16, 18]
-        group: [1, 2, 3]
+        group: [1, 2, 3, 4, 5, 6, 7]
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-        with:
-          node-version: ${{ matrix.node }}
-          check-latest: true
-
       - run: echo ${{needs.build.outputs.docsChange}}
 
       # https://github.com/actions/virtual-environments/issues/1187
@@ -510,16 +455,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: npm i -g pnpm@${PNPM_VERSION}
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      - run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
-        timeout-minutes: 10
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      # - run: npx @replayio/playwright install chromium
-      #   if: ${{needs.build.outputs.docsChange == 'nope'}}
-      - run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/7 >> /proc/1/fd/1"
         name: Run test/e2e (production)
         if: ${{needs.build.outputs.docsChange == 'nope'}}
         env:
@@ -571,15 +507,13 @@ jobs:
             18,
             19,
             20,
+            21,
+            22,
+            23,
+            24,
+            25,
           ]
     steps:
-      - name: Setup node
-        uses: actions/setup-node@v3
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-        with:
-          node-version: 16
-          check-latest: true
-
       - run: echo ${{needs.build.outputs.docsChange}}
 
       # https://github.com/actions/virtual-environments/issues/1187
@@ -599,16 +533,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: npm i -g pnpm@${PNPM_VERSION}
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      - run: npm i -g playwright-chromium@1.22.2 && npx playwright install-deps
-        timeout-minutes: 10
-        if: ${{needs.build.outputs.docsChange == 'nope'}}
-
-      # - run: npx @replayio/playwright install chromium
-      #   if: ${{needs.build.outputs.docsChange == 'nope'}}
-      - run: xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/20
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.22.2-focal /bin/bash -c "cd /work && ls && curl https://install-node.vercel.app/v16 | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} && xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/25 >> /proc/1/fd/1"
         if: ${{needs.build.outputs.docsChange == 'nope'}}
         # env:
         # RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testIntegration / Group ${{ matrix.group }}

diff --git a/package.json b/package.json
@@ -57,6 +57,7 @@
     "@edge-runtime/jest-environment": "2.0.0",
     "@fullhuman/postcss-purgecss": "1.3.0",
     "@mdx-js/loader": "^1.5.1",
+    "@mdx-js/react": "^1.6.18",
     "@next/bundle-analyzer": "workspace:*",
     "@next/env": "workspace:*",
     "@next/eslint-plugin-next": "workspace:*",
@@ -114,7 +115,9 @@
     "alex": "9.1.0",
     "amphtml-validator": "1.0.35",
     "async-sema": "3.0.1",
+    "body-parser": "1.20.1",
     "browserslist": "4.20.2",
+    "buffer": "5.6.0",
     "chalk": "5.0.1",
     "cheerio": "0.22.0",
     "cookie": "0.4.1",
@@ -140,6 +143,7 @@
     "express": "4.17.0",
     "faker": "5.5.3",
     "faunadb": "2.6.1",
+    "find-up": "4.1.0",
     "firebase": "7.14.5",
     "flat": "5.0.2",
     "form-data": "4.0.0",
@@ -149,15 +153,18 @@
     "gzip-size": "5.1.1",
     "html-validator": "5.1.18",
     "husky": "8.0.0",
+    "http-proxy": "1.18.1",
     "image-size": "0.9.3",
     "is-animated": "2.0.2",
     "isomorphic-unfetch": "3.0.0",
     "jest": "27.0.6",
     "jest-extended": "1.2.1",
+    "json5": "2.2.1",
     "ky": "0.19.1",
     "ky-universal": "0.6.0",
     "lerna": "4.0.0",
     "lint-staged": "10.1.7",
+    "lodash": "4.17.20",
     "lost": "8.3.1",
     "minimatch": "3.0.4",
     "moment": "^2.24.0",
@@ -197,7 +204,9 @@
     "selenium-webdriver": "4.0.0-beta.4",
     "semver": "7.3.7",
     "shell-quote": "1.7.3",
+    "strip-ansi": "6.0.0",
     "styled-components": "6.0.0-beta.5",
+    "styled-jsx": "5.1.0",
     "styled-jsx-plugin-postcss": "3.0.2",
     "swr": "2.0.0-rc.0",
     "tailwindcss": "1.1.3",
@@ -206,9 +215,12 @@
     "tsec": "0.2.1",
     "turbo": "1.6.3",
     "typescript": "4.8.2",
+    "unfetch": "4.2.0",
     "wait-port": "0.2.2",
+    "whatwg-fetch": "3.0.0",
     "webpack": "5.74.0",
-    "webpack-bundle-analyzer": "4.7.0"
+    "webpack-bundle-analyzer": "4.7.0",
+    "ws": "8.2.3"
   },
   "resolutions": {
     "browserslist": "4.20.2",