Improve test runner performance with turbo

.github/actions/next-stats-action/src/prepare/repo-setup.js

@@ -55,14 +55,13 @@ module.exports = (actionInfo) => {
       }
     },
     async linkPackages({ repoDir, nextSwcVersion }) {
-      let useTestPack = process.env.NEXT_TEST_PACK
+      const useTurbo = Boolean(process.env.NEXT_TEST_PACK)
 
-      if (useTestPack) {
-        execa.sync('pnpm', ['turbo', 'run', 'test-pack'], {
+      if (useTurbo) {
+        execa.sync('pnpm', ['test-pack-all'], {
           cwd: repoDir,
           env: { NEXT_SWC_VERSION: nextSwcVersion },
         })
-
         const pkgPaths = new Map()
         const pkgs = (await fs.readdir(path.join(repoDir, 'packages'))).filter(
           (item) => !item.startsWith('.')
@@ -87,7 +86,6 @@ module.exports = (actionInfo) => {
         })
         return pkgPaths
       } else {
-        // TODO: remove after next stable release (current v13.1.2)
         const pkgPaths = new Map()
         const pkgDatas = new Map()
         let pkgs

contributing/core/testing.md

@@ -79,3 +79,4 @@ When tests are run in CI and a test failure occurs we attempt to capture traces
 ### Profiling tests
 
 Add `NEXT_TEST_TRACE=1` to enable test profiling. It's useful for improving our testing infrastructure.
+Those traces can be visualized with `node scripts/trace-to-tree.mjs test/.trace/trace`.

package.json

@@ -23,7 +23,9 @@
     "test": "pnpm testheadless",
     "testonly": "jest --runInBand",
     "testheadless": "cross-env HEADLESS=true pnpm testonly",
-    "test-pack": "cross-env TS_NODE_TRANSPILE_ONLY=1 node --loader ts-node/esm scripts/test-pack-package.mts",
+    "test-pack-all": "pnpm exec-ts scripts/non-concurrent.mts packing-turbo pnpm turbo run test-pack --concurrency=100%",
+    "test-pack": "pnpm exec-ts scripts/test-pack-package.mts",
+    "exec-ts": "cross-env TS_NODE_TRANSPILE_ONLY=1 node --loader ts-node/esm",
     "genstats": "cross-env LOCAL_STATS=true node .github/actions/next-stats-action/src/index.js",
     "git-reset": "git reset --hard HEAD",
     "git-clean": "git clean -d -x -e node_modules -e packages -f",

run-tests.js

@@ -9,6 +9,7 @@ const { promisify } = require('util')
 const { Sema } = require('async-sema')
 const { spawn, exec: execOrig } = require('child_process')
 const { createNextInstall } = require('./test/lib/create-next-install')
+const { mockTrace } = require('./test/lib/mock-trace')
 const glob = promisify(_glob)
 const exec = promisify(execOrig)
 
@@ -43,11 +44,6 @@ const testFilters = {
   examples: 'examples/',
 }
 
-const mockTrace = () => ({
-  traceAsyncFn: (fn) => fn(mockTrace()),
-  traceChild: () => mockTrace(),
-})
-
 // which types we have configured to run separate
 const configuredTestTypes = Object.values(testFilters)
 

scripts/non-concurrent.mts

@@ -0,0 +1,71 @@
+import path from 'path'
+import execa from 'execa'
+import fs from 'fs-extra'
+import { fileURLToPath } from 'url'
+
+/**
+ * Make sure that script passed a arguments is not run concurrently.
+ * It will wait for other invocations with the same `operation-id` to finish before running.
+ *
+ * Usage:
+ *  node scripts/non-concurrent.mts [operation-id] [script with arguments]...
+ * Example:
+ *  node scripts/non-concurrent.mts test-pack node scripts/test-pack.mts --more --args
+ */
+
+const timeoutMs = 100
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+
+const cleanupFns: (() => void)[] = []
+
+const main = async () => {
+  const __dirname = fileURLToPath(new URL('.', import.meta.url))
+  const repoRoot = path.dirname(__dirname)
+  const operationId = process.argv[2]
+  const nonConcurrentFolder = path.join(
+    repoRoot,
+    'test',
+    'tmp',
+    'nonConcurrent'
+  )
+  const lockFolder = path.join(nonConcurrentFolder, `${operationId}.lock`)
+
+  while (true) {
+    // Create a file but throw if it already exists, use fs module
+    try {
+      await fs.ensureDir(nonConcurrentFolder)
+      await fs.mkdir(lockFolder)
+      cleanupFns.push(() => fs.rmdirSync(lockFolder, { recursive: true }))
+    } catch (err) {
+      if (err.code === 'EEXIST') {
+        console.log(`Waiting for other invocations to finish...`)
+        await sleep(timeoutMs)
+        continue
+      }
+      throw err
+    }
+
+    const proc = execa(process.argv[3], process.argv.slice(4))
+    proc.stdout?.pipe(process.stdout)
+    proc.stderr?.pipe(process.stderr)
+    await proc
+    return
+  }
+}
+
+const exitHandler = async () => {
+  for (const fn of cleanupFns) {
+    try {
+      fn()
+    } catch (err) {}
+  }
+  process.exit()
+}
+
+process.on('exit', exitHandler)
+process.on('SIGINT', exitHandler)
+process.on('SIGUSR1', exitHandler)
+process.on('SIGUSR2', exitHandler)
+process.on('uncaughtException', exitHandler)
+
+main()

scripts/test-pack-package.mts

@@ -40,6 +40,7 @@ const main = async () => {
     // There's a bug in `pnpm pack` where it will run
     // the prepublishOnly script and that will fail.
     // See https://github.com/pnpm/pnpm/issues/2941
+    // This is fixed in v7.26.0
     delete packageJson.scripts.prepublishOnly
   }
 

test/lib/create-next-install.js

@@ -21,6 +21,7 @@ async function createNextInstall({
     .traceAsyncFn(async (rootSpan) => {
       const tmpDir = await fs.realpath(process.env.NEXT_TEST_DIR || os.tmpdir())
       const origRepoDir = path.join(__dirname, '../../')
+      const useTurbo = Boolean(process.env.NEXT_TEST_PACK)
       const installDir = path.join(
         tmpDir,
         `next-install-${randomBytes(32).toString('hex')}${dirSuffix}`
@@ -30,85 +31,103 @@ async function createNextInstall({
       require('console').log(installDir)
 
       let pkgPaths = process.env.NEXT_TEST_PKG_PATHS
+      if (!useTurbo) {
+        if (pkgPaths) {
+          pkgPaths = new Map(JSON.parse(pkgPaths))
+          require('console').log('using provided pkg paths')
+        } else {
+          tmpRepoDir = path.join(
+            tmpDir,
+            `next-repo-${randomBytes(32).toString('hex')}${dirSuffix}`
+          )
+          require('console').log('Creating temp repo dir', tmpRepoDir)
 
-      if (pkgPaths) {
-        pkgPaths = new Map(JSON.parse(pkgPaths))
-        require('console').log('using provided pkg paths')
-      } else {
-        tmpRepoDir = path.join(
-          tmpDir,
-          `next-repo-${randomBytes(32).toString('hex')}${dirSuffix}`
-        )
-        require('console').log('Creating temp repo dir', tmpRepoDir)
-
-        await rootSpan
-          .traceChild('ensure swc binary')
-          .traceAsyncFn(async () => {
-            // ensure swc binary is present in the native folder if
-            // not already built
-            for (const folder of await fs.readdir(
-              path.join(origRepoDir, 'node_modules/@next')
-            )) {
-              if (folder.startsWith('swc-')) {
-                const swcPkgPath = path.join(
-                  origRepoDir,
-                  'node_modules/@next',
-                  folder
-                )
-                const outputPath = path.join(
-                  origRepoDir,
-                  'packages/next-swc/native'
-                )
-                await fs.copy(swcPkgPath, outputPath, {
-                  filter: (item) => {
-                    return (
-                      item === swcPkgPath ||
-                      (item.endsWith('.node') &&
-                        !fs.pathExistsSync(
-                          path.join(outputPath, path.basename(item))
-                        ))
-                    )
-                  },
-                })
-              }
-            }
-          })
-
-        for (const item of ['package.json', 'packages']) {
           await rootSpan
-            .traceChild(`copy ${item} to temp dir`)
+            .traceChild('ensure swc binary')
             .traceAsyncFn(async () => {
-              await fs.copy(
-                path.join(origRepoDir, item),
-                path.join(tmpRepoDir, item),
-                {
-                  filter: (item) => {
-                    return (
-                      !item.includes('node_modules') &&
-                      !item.includes('pnpm-lock.yaml') &&
-                      !item.includes('.DS_Store') &&
-                      // Exclude Rust compilation files
-                      !/next[\\/]build[\\/]swc[\\/]target/.test(item) &&
-                      !/next-swc[\\/]target/.test(item)
-                    )
-                  },
+              // ensure swc binary is present in the native folder if
+              // not already built
+              for (const folder of await fs.readdir(
+                path.join(origRepoDir, 'node_modules/@next')
+              )) {
+                if (folder.startsWith('swc-')) {
+                  const swcPkgPath = path.join(
+                    origRepoDir,
+                    'node_modules/@next',
+                    folder
+                  )
+                  const outputPath = path.join(
+                    origRepoDir,
+                    'packages/next-swc/native'
+                  )
+                  await fs.copy(swcPkgPath, outputPath, {
+                    filter: (item) => {
+                      return (
+                        item === swcPkgPath ||
+                        (item.endsWith('.node') &&
+                          !fs.pathExistsSync(
+                            path.join(outputPath, path.basename(item))
+                          ))
+                      )
+                    },
+                  })
                 }
-              )
+              }
             })
-        }
 
-        pkgPaths = await rootSpan.traceChild('linkPackages').traceAsyncFn(() =>
-          linkPackages({
-            repoDir: tmpRepoDir,
-          })
-        )
+          for (const item of ['package.json', 'packages']) {
+            await rootSpan
+              .traceChild(`copy ${item} to temp dir`)
+              .traceAsyncFn(async () => {
+                await fs.copy(
+                  path.join(origRepoDir, item),
+                  path.join(tmpRepoDir, item),
+                  {
+                    filter: (item) => {
+                      return (
+                        !item.includes('node_modules') &&
+                        !item.includes('pnpm-lock.yaml') &&
+                        !item.includes('.DS_Store') &&
+                        // Exclude Rust compilation files
+                        !/next[\\/]build[\\/]swc[\\/]target/.test(item) &&
+                        !/next-swc[\\/]target/.test(item)
+                      )
+                    },
+                  }
+                )
+              })
+          }
+
+          pkgPaths = await rootSpan
+            .traceChild('linkPackages')
+            .traceAsyncFn(() =>
+              linkPackages({
+                repoDir: tmpRepoDir,
+              })
+            )
+        }
+        if (onlyPackages) {
+          return pkgPaths
+        }
       }
+
       let combinedDependencies = dependencies
 
-      if (onlyPackages) {
-        return pkgPaths
-      }
       if (!(packageJson && packageJson.nextParamateSkipLocalDeps)) {
+        if (useTurbo) {
+          pkgPaths = await rootSpan
+            .traceChild('linkPackages')
+            .traceAsyncFn(() =>
+              linkPackages({
+                repoDir: useTurbo ? origRepoDir : tmpRepoDir,
+              })
+            )
+
+          if (onlyPackages) {
+            return pkgPaths
+          }
+        }
+
         combinedDependencies = {
           next: pkgPaths.get('next'),
           ...Object.keys(dependencies).reduce((prev, pkg) => {
@@ -178,6 +197,7 @@ async function createNextInstall({
           tmpRepoDir,
         }
       }
+
       return installDir
     })
 }

test/lib/mock-trace.js

@@ -0,0 +1,6 @@
+const mockTrace = () => ({
+  traceAsyncFn: (fn) => fn(mockTrace()),
+  traceChild: () => mockTrace(),
+})
+
+module.exports = { mockTrace }

turbo.json

@@ -31,13 +31,17 @@
     },
     "typescript": {},
     "test-pack": {
-      "dependsOn": ["^test-pack", "test-pack-global-deps"],
+      "dependsOn": ["test-pack-global-deps", "test-pack-gitignored-deps"],
       "outputs": ["packed-*.tgz"],
       "env": ["NEXT_SWC_VERSION"]
     },
     "test-pack-global-deps": {
       "inputs": ["../../scripts/test-pack-package.mts", "../../package.json"],
       "cache": false
+    },
+    "test-pack-gitignored-deps": {
+      "inputs": ["dist/**"],
+      "cache": false
     }
   }
 }