ci(workflow): enable test trace upload

.github/workflows/build_and_test.yml

@@ -23,6 +23,7 @@ env:
   # canary next-swc binaries in the monorepo
   NEXT_SKIP_NATIVE_POSTINSTALL: 1
   DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DATADOG_TRACE_NEXTJS_TEST: 'true'
   DD_ENV: 'ci'
   TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
   NEXT_TEST_JOB: 1
@@ -146,6 +147,22 @@ jobs:
       afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
     secrets: inherit
 
+  report-test-results:
+    needs: ['test-dev', 'test-prod', 'test-integration']
+    runs-on: [self-hosted, linux, x64]
+    name: upload test trace to datadog
+    # We'll not block the CI if the upload fails
+    continue-on-error: true
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: Test trace reports
+          path: test/test-junit-report
+      - run: |
+          npm install -g junit-report-merger @datadog/datadog-ci
+          jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml"
+          DD_ENV=ci datadog-ci junit upload --git-repository-url https://github.com/vercel/next.js --service nextjs ./nextjs-test-result-junit.xml
+
   tests-pass:
     needs:
       [

.github/workflows/build_reusable.yml

@@ -45,6 +45,7 @@ env:
   # canary next-swc binaries in the monorepo
   NEXT_SKIP_NATIVE_POSTINSTALL: 1
   DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DATADOG_TRACE_NEXTJS_TEST: 'true'
   DD_ENV: 'ci'
   TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
   NEXT_TEST_JOB: 1
@@ -126,3 +127,11 @@ jobs:
         with:
           name: turbo run summary
           path: .turbo/runs
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ inputs.afterBuild }}
+        with:
+          name: Test trace reports
+          path: test/test-junit-report
+          if-no-files-found: ignore

jest.config.js

@@ -25,5 +25,29 @@ const customJestConfig = {
   },
 }
 
+// Check if the environment variable is set to enable test trace,
+// Insert a reporter to generate a junit report to upload.
+// This won't count for the retry to avoid duplicated test being reported twice
+// - which means our test trace will report test results for the flaky test as failed without retry.
+const shouldEnableTestTrace =
+  process.env.DATADOG_API_KEY &&
+  process.env.DATADOG_TRACE_NEXTJS_TEST &&
+  !process.env.IS_RETRY
+
+if (shouldEnableTestTrace) {
+  if (!customJestConfig.reporters) {
+    customJestConfig.reporters = ['default']
+  }
+  customJestConfig.reporters.push([
+    'jest-junit',
+    {
+      outputDirectory: '<rootDir>/test-junit-report',
+      reportTestSuiteErrors: 'true',
+      uniqueOutputName: 'true',
+      outputName: 'nextjs-test-junit',
+    },
+  ])
+}
+
 // createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
 module.exports = createJestConfig(customJestConfig)

package.json

@@ -152,6 +152,7 @@
     "isomorphic-unfetch": "3.0.0",
     "jest": "27.0.6",
     "jest-extended": "1.2.1",
+    "jest-junit": "16.0.0",
     "json5": "2.2.3",
     "ky": "0.19.1",
     "ky-universal": "0.6.0",

run-tests.js

@@ -322,33 +322,6 @@ async function main() {
 
       const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
 
-      let shouldEnableTestTrace = false
-      let traceInitPath = null
-
-      try {
-        traceInitPath = require.resolve('dd-trace/ci/init')
-        // Enable trace only if tracer (dd-trace) can be resolved
-        shouldEnableTestTrace =
-          process.env.DATADOG_API_KEY && process.env.DATADOG_TRACE_NEXTJS_TEST
-      } catch (e) {
-        shouldEnableTestTrace = false
-      }
-
-      const traceEnv =
-        shouldEnableTestTrace && traceInitPath
-          ? {
-              DD_API_KEY: process.env.DATADOG_API_KEY,
-              DD_CIVISIBILITY_AGENTLESS_ENABLED: 1,
-              DD_ENV: 'ci',
-              DD_SERVICE: 'nextjs',
-              NODE_OPTIONS: `-r ${traceInitPath}`,
-            }
-          : {}
-
-      if (shouldEnableTestTrace) {
-        console.log(`Running test with Datadog tracing enabled`)
-      }
-
       const child = spawn(
         jestPath,
         [
@@ -368,7 +341,7 @@ async function main() {
           stdio: ['ignore', 'pipe', 'pipe'],
           env: {
             ...process.env,
-            ...traceEnv,
+            IS_RETRY: isRetry ? 'true' : undefined,
             RECORD_REPLAY: shouldRecordTestWithReplay,
             // run tests in headless mode by default
             HEADLESS: 'true',

test/.gitignore

@@ -2,3 +2,4 @@
 .vscode
 
 e2e/**/tsconfig.json
+test-junit-report/