Parallelize Turbopack CI tests (vercel#50292)

### What? Parallel jobs ### Why? Timeouts otherwise Co-authored-by: JJ Kasper <22380829+ijjk@users.noreply.github.com>
hydRAnger · Jun 12, 2023 · 01a19d5 · 01a19d5
1 parent 6ff1ec5
commit 01a19d5
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 39 deletions.
diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml
@@ -833,7 +833,7 @@ jobs:
       fail-fast: false
       matrix:
         node: [18]
-        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        group: [1, 2, 3, 4, 5]
     steps:
       - run: echo "${{needs.build.outputs.docsChange}}"
 
@@ -855,7 +855,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -g ${{ matrix.group }}/10 >> /proc/1/fd/1"
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -g ${{ matrix.group }}/5 >> /proc/1/fd/1"
         name: Run test/development
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
@@ -878,7 +878,7 @@ jobs:
       fail-fast: false
       matrix:
         node: [18]
-        group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        group: [1, 2, 3, 4, 5]
     steps:
       - run: echo "${{needs.build.outputs.docsChange}}"
 
@@ -900,7 +900,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/10 >> /proc/1/fd/1"
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/5 >> /proc/1/fd/1"
         name: Run test/e2e (dev)
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
@@ -932,7 +932,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        group: [1]
+        group: [1, 2, 3, 4, 5]
     steps:
       - run: echo "${{needs.build.outputs.docsChange}}"
 
@@ -954,7 +954,7 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings >> /proc/1/fd/1"
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/5 >> /proc/1/fd/1"
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
       - name: Upload test trace

diff --git a/packages/next-swc/crates/next-dev-tests/tests-manifest.js b/packages/next-swc/crates/next-dev-tests/tests-manifest.js
@@ -91,7 +91,6 @@ const enabledTests = [
   'test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts',
   'test/e2e/ssr-react-context/index.test.ts',
   'test/e2e/styled-jsx/index.test.ts',
-  'test/e2e/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
   'test/e2e/test-utils-tests/basic/basic.test.ts',
   'test/e2e/trailingslash-with-rewrite/index.test.ts',
   'test/e2e/transpile-packages/index.test.ts',
@@ -121,39 +120,24 @@ const enabledTests = [
   'test/integration/app-functional/test/index.test.js',
   'test/integration/app-tree/test/index.test.js',
   'test/integration/app-types/app-types.test.js',
-  'test/integration/auto-export-error-bail/test/index.test.js',
-  'test/integration/auto-export-query-error/test/index.test.js',
+  // TODO: should babel be tested with turbopack?
   'test/integration/babel-custom/test/index.test.js',
   'test/integration/bigint/test/index.test.js',
-  'test/integration/build-output/test/index.test.js',
-  'test/integration/build-trace-extra-entries-turbo/test/index.test.js',
-  'test/integration/build-trace-extra-entries/test/index.test.js',
-  'test/integration/build-warnings/test/index.test.js',
   'test/integration/catches-missing-getStaticProps/test/index.test.js',
   'test/integration/chunking/test/index.test.js',
-  'test/integration/clean-distdir/test/index.test.js',
   'test/integration/client-404/test/index.test.js',
   'test/integration/client-navigation-a11y/test/index.test.js',
   'test/integration/client-shallow-routing/test/index.test.js',
   'test/integration/config-experimental-warning/test/index.test.js',
   'test/integration/config-promise-error/test/index.test.js',
   'test/integration/config-resolve-alias/test/index.test.js',
-  'test/integration/config-schema-check/test/index.test.js',
-  'test/integration/config-syntax-error/test/index.test.js',
-  'test/integration/config-validation/test/index.test.ts',
-  'test/integration/conflicting-ssg-paths/test/index.test.js',
-  'test/integration/create-next-app/index.test.ts',
-  'test/integration/create-next-app/templates.test.ts',
-  'test/integration/critical-css/test/index.test.js',
   'test/integration/css-customization/test/index.test.js',
   'test/integration/css-features/test/index.test.js',
   'test/integration/css-minify/test/index.test.js',
   'test/integration/custom-error-page-exception/test/index.test.js',
-  'test/integration/custom-server-types/test/index.test.js',
   'test/integration/dedupes-scripts/test/index.test.js',
   'test/integration/development-hmr-refresh/test/index.test.js',
   'test/integration/disable-js/test/index.test.js',
-  'test/integration/dist-dir/test/index.test.js',
   'test/integration/document-file-dependencies/test/index.test.js',
   'test/integration/document-head-warnings/test/index.test.js',
   'test/integration/duplicate-pages/test/index.test.js',
@@ -166,19 +150,6 @@ const enabledTests = [
   'test/integration/errors-on-output-to-public/test/index.test.js',
   'test/integration/errors-on-output-to-static/test/index.test.js',
   'test/integration/eslint/test/index.test.js',
-  'test/integration/export-404/test/index.test.js',
-  'test/integration/export-default-map/test/index.test.js',
-  'test/integration/export-dynamic-pages/test/index.test.js',
-  'test/integration/export-fallback-true-error/test/index.test.js',
-  'test/integration/export-getInitialProps-warn/test/index.test.js',
-  'test/integration/export-image-default/test/index.test.js',
-  'test/integration/export-image-loader-legacy/test/index.test.js',
-  'test/integration/export-image-loader/test/index.test.js',
-  'test/integration/export-index-not-found-gsp/test/index.test.ts',
-  'test/integration/export-intent/test/index.test.js',
-  'test/integration/export-no-build/test/index.test.js',
-  'test/integration/export-progress-status-message/test/index.test.js',
-  'test/integration/export-subfolders/test/index.test.js',
   'test/integration/externalize-next-server/test/index.test.js',
   'test/integration/externals-esm-loose/test/index.test.js',
   'test/integration/externals-esm/test/index.test.js',
@@ -215,7 +186,6 @@ const enabledTests = [
   'test/integration/link-ref/test/index.test.js',
   'test/integration/link-with-multiple-child/test/index.test.js',
   'test/integration/link-without-router/test/index.test.js',
-  'test/integration/middleware-build-errors/test/index.test.js',
   'test/integration/middleware-overrides-node.js-api/test/index.test.ts',
   'test/integration/middleware-prefetch/tests/index.test.js',
   'test/integration/middleware-src/test/index.test.js',
@@ -293,13 +263,65 @@ const enabledTests = [
   'test/integration/typescript-ignore-errors/test/index.test.js',
   'test/integration/webpack-config-extensionalias/test/index.test.js',
   'test/integration/webpack-config-mainjs/test/index.test.js',
-  'test/integration/webpack-require-hook/test/index.test.js',
-  'test/integration/with-electron/test/index.test.js',
 ]
 
 // Tests that are currently disabled with Turbopack in CI.
 // Any tests not listed in here are assumed to be enabled.
 const disabledTests = [
+  // these are build specific
+  'test/integration/auto-export-error-bail/test/index.test.js',
+  'test/integration/auto-export-query-error/test/index.test.js',
+
+  // these are build specific
+  'test/integration/build-output/test/index.test.js',
+  'test/integration/build-trace-extra-entries-turbo/test/index.test.js',
+  'test/integration/build-trace-extra-entries/test/index.test.js',
+  'test/integration/build-warnings/test/index.test.js',
+
+  // next build specific
+  'test/integration/config-schema-check/test/index.test.js',
+  'test/integration/config-syntax-error/test/index.test.js',
+  'test/integration/config-validation/test/index.test.ts',
+  'test/integration/conflicting-ssg-paths/test/index.test.js',
+
+  // these are already tested against turbopack explicitly in the suite
+  'test/integration/create-next-app/index.test.ts',
+  'test/integration/create-next-app/templates.test.ts',
+
+  // next build specific
+  'test/integration/critical-css/test/index.test.js',
+
+  // custom-server isn't compatible with turbopack why is it here?
+  'test/integration/custom-server-types/test/index.test.js',
+
+  // Clean dist dir is a production only test should it be here?
+  'test/integration/clean-distdir/test/index.test.js',
+
+  // this is next build specific why is it here?
+  'test/integration/dist-dir/test/index.test.js',
+
+  // these are next build specific
+  'test/integration/export-404/test/index.test.js',
+  'test/integration/export-default-map/test/index.test.js',
+  'test/integration/export-dynamic-pages/test/index.test.js',
+  'test/integration/export-fallback-true-error/test/index.test.js',
+  'test/integration/export-getInitialProps-warn/test/index.test.js',
+  'test/integration/export-image-default/test/index.test.js',
+  'test/integration/export-image-loader-legacy/test/index.test.js',
+  'test/integration/export-image-loader/test/index.test.js',
+  'test/integration/export-index-not-found-gsp/test/index.test.ts',
+  'test/integration/export-intent/test/index.test.js',
+  'test/integration/export-no-build/test/index.test.js',
+  'test/integration/export-progress-status-message/test/index.test.js',
+  'test/integration/export-subfolders/test/index.test.js',
+
+  // build specific
+  'test/integration/middleware-build-errors/test/index.test.js',
+  'test/integration/webpack-require-hook/test/index.test.js',
+  // this is next build and next export specific
+  'test/integration/with-electron/test/index.test.js',
+  'test/e2e/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
+
   'test/development/acceptance-app/app-hmr-changes.test.ts',
   'test/development/acceptance-app/component-stack.test.ts',
   'test/development/acceptance-app/editor-links.test.ts',