Add tracing for testing tools (#44046)

.github/actions/next-stats-action/src/index.js

@@ -135,10 +135,10 @@ if (!allowedActions.has(actionInfo.actionName) && !actionInfo.isRelease) {
 
       logger(`Linking packages in ${dir}`)
       const isMainRepo = dir === mainRepoDir
-      const pkgPaths = await linkPackages(
-        dir,
-        isMainRepo ? mainNextSwcVersion : undefined
-      )
+      const pkgPaths = await linkPackages({
+        repoDir: dir,
+        nextSwcPkg: isMainRepo ? mainNextSwcVersion : undefined,
+      })
 
       if (isMainRepo) mainRepoPkgPaths = pkgPaths
       else diffRepoPkgPaths = pkgPaths

.github/actions/next-stats-action/src/prepare/repo-setup.js

@@ -5,6 +5,11 @@ const { remove } = require('fs-extra')
 const logger = require('../util/logger')
 const semver = require('semver')
 
+const mockTrace = () => ({
+  traceAsyncFn: (fn) => fn(mockTrace()),
+  traceChild: () => mockTrace(),
+})
+
 module.exports = (actionInfo) => {
   return {
     async cloneRepo(repoPath = '', dest = '') {
@@ -53,96 +58,117 @@ module.exports = (actionInfo) => {
         }
       }
     },
-    async linkPackages(repoDir = '', nextSwcPkg) {
-      const pkgPaths = new Map()
-      const pkgDatas = new Map()
-      let pkgs
+    async linkPackages({ repoDir = '', nextSwcPkg, parentSpan }) {
+      const rootSpan = parentSpan
+        ? parentSpan.traceChild('linkPackages')
+        : mockTrace()
 
-      try {
-        pkgs = await fs.readdir(path.join(repoDir, 'packages'))
-      } catch (err) {
-        if (err.code === 'ENOENT') {
-          console.log('no packages to link')
-          return pkgPaths
+      return await rootSpan.traceAsyncFn(async () => {
+        const pkgPaths = new Map()
+        const pkgDatas = new Map()
+        let pkgs
+
+        try {
+          pkgs = await fs.readdir(path.join(repoDir, 'packages'))
+        } catch (err) {
+          if (err.code === 'ENOENT') {
+            console.log('no packages to link')
+            return pkgPaths
+          }
+          throw err
         }
-        throw err
-      }
 
-      for (const pkg of pkgs) {
-        const pkgPath = path.join(repoDir, 'packages', pkg)
-        const packedPkgPath = path.join(pkgPath, `${pkg}-packed.tgz`)
+        await rootSpan
+          .traceChild('prepare packages for packing')
+          .traceAsyncFn(async () => {
+            for (const pkg of pkgs) {
+              const pkgPath = path.join(repoDir, 'packages', pkg)
+              const packedPkgPath = path.join(pkgPath, `${pkg}-packed.tgz`)
 
-        const pkgDataPath = path.join(pkgPath, 'package.json')
-        if (!fs.existsSync(pkgDataPath)) {
-          console.log(`Skipping ${pkgDataPath}`)
-          continue
-        }
-        const pkgData = require(pkgDataPath)
-        const { name } = pkgData
-        pkgDatas.set(name, {
-          pkgDataPath,
-          pkg,
-          pkgPath,
-          pkgData,
-          packedPkgPath,
-        })
-        pkgPaths.set(name, packedPkgPath)
-      }
+              const pkgDataPath = path.join(pkgPath, 'package.json')
+              if (!fs.existsSync(pkgDataPath)) {
+                console.log(`Skipping ${pkgDataPath}`)
+                continue
+              }
+              const pkgData = require(pkgDataPath)
+              const { name } = pkgData
+              pkgDatas.set(name, {
+                pkgDataPath,
+                pkg,
+                pkgPath,
+                pkgData,
+                packedPkgPath,
+              })
+              pkgPaths.set(name, packedPkgPath)
+            }
 
-      for (const pkg of pkgDatas.keys()) {
-        const { pkgDataPath, pkgData } = pkgDatas.get(pkg)
+            for (const pkg of pkgDatas.keys()) {
+              const { pkgDataPath, pkgData } = pkgDatas.get(pkg)
 
-        for (const pkg of pkgDatas.keys()) {
-          const { packedPkgPath } = pkgDatas.get(pkg)
-          if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue
-          pkgData.dependencies[pkg] = packedPkgPath
-        }
-        // make sure native binaries are included in local linking
-        if (pkg === '@next/swc') {
-          if (!pkgData.files) {
-            pkgData.files = []
-          }
-          pkgData.files.push('native')
-          console.log(
-            'using swc binaries: ',
-            await exec(`ls ${path.join(path.dirname(pkgDataPath), 'native')}`)
-          )
-        }
-        if (pkg === 'next') {
-          if (nextSwcPkg) {
-            Object.assign(pkgData.dependencies, nextSwcPkg)
-          } else {
-            if (pkgDatas.get('@next/swc')) {
-              pkgData.dependencies['@next/swc'] =
-                pkgDatas.get('@next/swc').packedPkgPath
-            } else {
-              pkgData.files.push('native')
+              for (const pkg of pkgDatas.keys()) {
+                const { packedPkgPath } = pkgDatas.get(pkg)
+                if (!pkgData.dependencies || !pkgData.dependencies[pkg])
+                  continue
+                pkgData.dependencies[pkg] = packedPkgPath
+              }
+
+              // make sure native binaries are included in local linking
+              if (pkg === '@next/swc') {
+                if (!pkgData.files) {
+                  pkgData.files = []
+                }
+                pkgData.files.push('native')
+                console.log(
+                  'using swc binaries: ',
+                  await exec(
+                    `ls ${path.join(path.dirname(pkgDataPath), 'native')}`
+                  )
+                )
+              }
+
+              if (pkg === 'next') {
+                if (nextSwcPkg) {
+                  Object.assign(pkgData.dependencies, nextSwcPkg)
+                } else {
+                  if (pkgDatas.get('@next/swc')) {
+                    pkgData.dependencies['@next/swc'] =
+                      pkgDatas.get('@next/swc').packedPkgPath
+                  } else {
+                    pkgData.files.push('native')
+                  }
+                }
+              }
+
+              await fs.writeFile(
+                pkgDataPath,
+                JSON.stringify(pkgData, null, 2),
+                'utf8'
+              )
             }
-          }
-        }
-        await fs.writeFile(
-          pkgDataPath,
-          JSON.stringify(pkgData, null, 2),
-          'utf8'
-        )
-      }
+          })
 
-      // wait to pack packages until after dependency paths have been updated
-      // to the correct versions
-      await Promise.all(
-        Array.from(pkgDatas.keys()).map(async (pkgName) => {
-          const { pkg, pkgPath } = pkgDatas.get(pkgName)
-          await exec(`cd ${pkgPath} && yarn pack -f ${pkg}-packed.tgz`, true, {
-            env: {
-              // Yarn installed through corepack will not run in pnpm project without this env var set
-              // This var works for corepack >=0.15.0
-              COREPACK_ENABLE_STRICT: '0',
-            },
+        // wait to pack packages until after dependency paths have been updated
+        // to the correct versions
+        await rootSpan
+          .traceChild('packing packages')
+          .traceAsyncFn(async (packingSpan) => {
+            await Promise.all(
+              Array.from(pkgDatas.keys()).map(async (pkgName) => {
+                await packingSpan
+                  .traceChild(`pack ${pkgName}`)
+                  .traceAsyncFn(async () => {
+                    const { pkg, pkgPath } = pkgDatas.get(pkgName)
+                    await exec(
+                      `cd ${pkgPath} && yarn pack -f '${pkg}-packed.tgz'`,
+                      true
+                    )
+                  })
+              })
+            )
           })
-        })
-      )
 
-      return pkgPaths
+        return pkgPaths
+      })
     },
   }
 }

.gitignore

@@ -28,6 +28,7 @@ test/**/tsconfig.json
 .DS_Store
 /e2e-tests
 test/tmp/**
+test/.trace
 
 # Editors
 **/.idea

contributing/core/testing.md

@@ -76,3 +76,7 @@ Some test-specific environment variables can be used to help debug isolated test
 ### Debugging
 
 When tests are run in CI and a test failure occurs we attempt to capture traces of the playwright run to make debugging the failure easier. A test-trace artifact should be uploaded after the workflow completes which can be downloaded, unzipped, and then inspected with `pnpm playwright show-trace ./path/to/trace`
+
+### Profiling tests
+
+Add `NEXT_TEST_TRACE=1` to enable test profiling. It's useful for improving our testing infrastructure.

packages/next/trace/trace.ts

@@ -93,17 +93,17 @@ export class Span {
     this.attrs[key] = String(value)
   }
 
-  traceFn<T>(fn: () => T): T {
+  traceFn<T>(fn: (span: Span) => T): T {
     try {
-      return fn()
+      return fn(this)
     } finally {
       this.stop()
     }
   }
 
-  async traceAsyncFn<T>(fn: () => T | Promise<T>): Promise<T> {
+  async traceAsyncFn<T>(fn: (span: Span) => T | Promise<T>): Promise<T> {
     try {
-      return await fn()
+      return await fn(this)
     } finally {
       this.stop()
     }

run-tests.js

@@ -29,6 +29,11 @@ const testFilters = {
   development: 'development/',
 }
 
+const mockTrace = () => ({
+  traceAsyncFn: (fn) => fn(mockTrace()),
+  traceChild: () => mockTrace(),
+})
+
 // which types we have configured to run separate
 const configuredTestTypes = Object.values(testFilters)
 
@@ -223,8 +228,11 @@ async function main() {
     console.log('Creating Next.js install for isolated tests')
     const reactVersion = process.env.NEXT_TEST_REACT_VERSION || 'latest'
     const testStarter = await createNextInstall({
-      react: reactVersion,
-      'react-dom': reactVersion,
+      parentSpan: mockTrace(),
+      dependencies: {
+        react: reactVersion,
+        'react-dom': reactVersion,
+      },
     })
     process.env.NEXT_TEST_STARTER = testStarter
   }

scripts/trace-next-server.js

@@ -38,7 +38,7 @@ async function main() {
   console.log('using repodir', repoDir)
   await fs.ensureDir(workDir)
 
-  const pkgPaths = await linkPackages(repoDir)
+  const pkgPaths = await linkPackages({ repoDir })
 
   await fs.writeFile(
     path.join(workDir, 'package.json'),

test/e2e/test-utils-tests/basic/basic.test.ts

@@ -0,0 +1,24 @@
+import { createNext, FileRef } from 'e2e-utils'
+import { NextInstance } from 'test/lib/next-modes/base'
+import { fetchViaHTTP } from 'next-test-utils'
+
+describe('createNext', () => {
+  let next: NextInstance
+
+  beforeAll(async () => {
+    next = await createNext({
+      files: new FileRef(__dirname),
+      dependencies: {
+        typescript: 'latest',
+        '@types/react': 'latest',
+        '@types/node': 'latest',
+      },
+    })
+  })
+  afterAll(() => next.destroy())
+
+  it('should work', async () => {
+    const res = await fetchViaHTTP(next.url, '/')
+    expect(await res.text()).toContain('Hello World')
+  })
+})

test/e2e/test-utils-tests/basic/pages/index.tsx

@@ -0,0 +1,3 @@
+export default function Page() {
+  return <h1>Hello World</h1>
+}