test(turbopack): run next.js integration tests with denylist

.github/workflows/build_test_deploy.yml

@@ -812,9 +812,8 @@ jobs:
       - run: xvfb-run node run-tests.js test/integration/with-electron/test/index.test.js
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
-  # A job to run sets of tests with turbopack enabled. These tests are considered as `stable`,
-  # that running with turbopack should always pass.
-  testTurbopack:
+  # A job to run sets of devlopment tests with turbopack enabled.
+  testDevTurbopack:
     name: Test Development (Turbopack)
     runs-on: ubuntu-latest
     needs: [build, build-native-test]
@@ -832,6 +831,10 @@ jobs:
 
     strategy:
       fail-fast: false
+      matrix:
+        node: [18]
+        # Currently turbopack tests does not utilize groups yet
+        group: [1]
     steps:
       - run: echo "${{needs.build.outputs.docsChange}}"
 
@@ -853,10 +856,119 @@ jobs:
           name: next-swc-test-binary
           path: packages/next-swc/native
 
-      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -c 1 >> /proc/1/fd/1"
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings >> /proc/1/fd/1"
         name: Run test/development
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
+  testDevE2ETurbopack:
+    name: Test Development (E2E) (Turbopack)
+    runs-on: ubuntu-latest
+    needs: [build, build-native-test]
+    timeout-minutes: 35
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
+      # Enabling backtrace will makes snapshot tests fail
+      RUST_BACKTRACE: 0
+      # Path to the custom next-swc bindings.
+      NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
+      # Glob pattern to run specific tests with --turbo.
+      NEXT_DEV_TEST_GLOB: '*'
+      NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
+    strategy:
+      fail-fast: false
+      matrix:
+        node: [18]
+        # Currently turbopack tests does not utilize groups yet
+        group: [1]
+    steps:
+      - run: echo "${{needs.build.outputs.docsChange}}"
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v3
+        timeout-minutes: 5
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}
+
+      - uses: actions/download-artifact@v3
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        with:
+          name: next-swc-test-binary
+          path: packages/next-swc/native
+
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings >> /proc/1/fd/1"
+        name: Run test/e2e (dev)
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+
+      - name: Upload test trace
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-trace
+          if-no-files-found: ignore
+          retention-days: 2
+          path: |
+            test/traces
+
+  testIntegrationTurbopack:
+    name: Test Integration (Turbopack)
+    runs-on: ubuntu-latest
+    needs: [build, build-native-test]
+    timeout-minutes: 35
+    env:
+      NEXT_TELEMETRY_DISABLED: 1
+      TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
+      # Enabling backtrace will makes snapshot tests fail
+      RUST_BACKTRACE: 0
+      # Path to the custom next-swc bindings.
+      NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
+      # Glob pattern to run specific tests with --turbo.
+      NEXT_DEV_TEST_GLOB: '*'
+      NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1]
+    steps:
+      - run: echo "${{needs.build.outputs.docsChange}}"
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v3
+        timeout-minutes: 5
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}
+
+      - uses: actions/download-artifact@v3
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+        with:
+          name: next-swc-test-binary
+          path: packages/next-swc/native
+
+      - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings >> /proc/1/fd/1"
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+
+      - name: Upload test trace
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-trace
+          if-no-files-found: ignore
+          retention-days: 2
+          path: |
+            test/traces
+
   testsPass:
     name: thank you, next
     runs-on: ubuntu-latest

packages/next-swc/crates/next-dev-tests/sync-tests-manifest.js

@@ -8,10 +8,11 @@ const path = require('path')
 
 const generateManifest = (enabledTests, disabledTests) => `
 // Tests that are currently enabled with Turbopack in CI.
-// Add new test when Turbopack updates to fix / implement a feature.
+// This list is not actively used, more of high level pictures of what tests are enabled.
 const enabledTests = ${enabledTests}
 
 // Tests that are currently disabled with Turbopack in CI.
+// Any tests not listed in here are assumed to be enabled.
 const disabledTests = ${disabledTests}
 
 module.exports = {
@@ -30,7 +31,7 @@ const main = async () => {
     enabledTests = manifest.enabledTests
     disabledTests = manifest.disabledTests
   } else {
-    throw new Error('a')
+    throw new Error('Manifest should exists')
   }
 
   // Collect all test files

packages/next-swc/crates/next-dev-tests/tests-manifest.js

@@ -1,5 +1,5 @@
 // Tests that are currently enabled with Turbopack in CI.
-// Add new test when Turbopack updates to fix / implement a feature.
+// This list is not actively used, more of high level pictures of what tests are enabled.
 const enabledTests = [
   'test/development/acceptance-app/dynamic-error.test.ts',
   'test/development/basic/legacy-decorators.test.ts',
@@ -9,6 +9,7 @@ const enabledTests = [
 ]
 
 // Tests that are currently disabled with Turbopack in CI.
+// Any tests not listed in here are assumed to be enabled.
 const disabledTests = [
   'test/development/acceptance-app/app-hmr-changes.test.ts',
   'test/development/acceptance-app/component-stack.test.ts',
@@ -76,6 +77,7 @@ const disabledTests = [
   'test/e2e/404-page-router/index.test.ts',
   'test/e2e/app-dir-legacy-edge-runtime-config/index.test.ts',
   'test/e2e/app-dir/_allow-underscored-root-directory/_allow-underscored-root-directory.test.ts',
+  'test/e2e/app-dir/actions/app-action-export.test.ts',
   'test/e2e/app-dir/actions/app-action.test.ts',
   'test/e2e/app-dir/app-a11y/index.test.ts',
   'test/e2e/app-dir/app-alias/app-alias.test.ts',
@@ -108,6 +110,7 @@ const disabledTests = [
   'test/e2e/app-dir/dynamic-href/dynamic-href.test.ts',
   'test/e2e/app-dir/dynamic/dynamic.test.ts',
   'test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts',
+  'test/e2e/app-dir/error-boundary-and-not-found-linking/error-boundary-and-not-found-linking.test.ts',
   'test/e2e/app-dir/front-redirect-issue/front-redirect-issue.test.ts',
   'test/e2e/app-dir/global-error/global-error.test.ts',
   'test/e2e/app-dir/hooks/hooks.test.ts',
@@ -672,6 +675,7 @@ const disabledTests = [
   'test/unit/esm-interpolate/esm-interpolate.test.tsx',
   'test/unit/find-config.test.ts',
   'test/unit/find-page-file.test.ts',
+  'test/unit/get-module-build-info.test.ts',
   'test/unit/get-node-options-without-inspect.test.ts',
   'test/unit/get-page-static-infos.test.ts',
   'test/unit/get-project-dir.test.ts',

run-tests.js

@@ -12,11 +12,11 @@ const { createNextInstall } = require('./test/lib/create-next-install')
 const glob = promisify(_glob)
 const exec = promisify(execOrig)
 
-// Try to read an external array-based json to filter tests to be executed.
+// Try to read an external array-based json to filter tests to be allowed / or disallowed.
 // If process.argv contains a test to be executed, this'll append it to the list.
 const externalTestsFilterLists = process.env.NEXT_EXTERNAL_TESTS_FILTERS
-  ? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS).enabledTests
-  : []
+  ? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS)
+  : { enabledTests: [], disabledTests: [] }
 const timings = []
 const DEFAULT_NUM_RETRIES = os.platform() === 'win32' ? 2 : 1
 const DEFAULT_CONCURRENCY = 2
@@ -143,9 +143,7 @@ async function main() {
 
   console.log('Running tests with concurrency:', concurrency)
 
-  let tests = process.argv
-    .filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
-    .concat(externalTestsFilterLists)
+  let tests = process.argv.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
   let prevTimings
 
   if (tests.length === 0) {
@@ -190,6 +188,17 @@ async function main() {
     }
   }
 
+  // If there are external manifest contains list of tests, apply it to the test lists.
+  // Specifically, we filters out `disabledTests` from named export of the manifest.
+  if (externalTestsFilterLists?.disabledTests.length > 0) {
+    tests = tests.filter(
+      (test) =>
+        !externalTestsFilterLists.disabledTests.some((disabled) =>
+          disabled.includes(test)
+        )
+    )
+  }
+
   let testNames = [
     ...new Set(
       tests.map((f) => {