From 6a2fd8f004efb1215d2bc6ac6fe97fc0efb1a054 Mon Sep 17 00:00:00 2001
From: Moshe Atlow <moshe@atlow.co.il>
Date: Mon, 15 Aug 2022 12:12:11 +0300
Subject: [PATCH] test_runner: support programmatically running `--test`

---
 doc/api/test.md                     |  23 +++++
 lib/internal/main/test_runner.js    | 140 +------------------------
 lib/internal/test_runner/harness.js |  12 ++-
 lib/internal/test_runner/runner.js  | 154 ++++++++++++++++++++++++++++
 lib/internal/test_runner/test.js    |   2 +-
 lib/test.js                         |  10 +-
 6 files changed, 192 insertions(+), 149 deletions(-)
 create mode 100644 lib/internal/test_runner/runner.js

diff --git a/doc/api/test.md b/doc/api/test.md
index 551259d314bb70..030f39aed12a4f 100644
--- a/doc/api/test.md
+++ b/doc/api/test.md
@@ -316,6 +316,29 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## `runFiles([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object} Configuration options for running test files. The following
+  properties are supported:
+  * `concurrency` {number|boolean} If a number is provided,
+    then that many files would run in parallel.
+    If truthy, it would run (number of cpu cores - 1)
+    files in parallel.
+    If falsy, it would only run one file at a time.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `true`.
+  * `files`: {Array} An array containing the list of files to run.
+    **Default** matching files from [test runner execution model][].
+  * `signal` {AbortSignal} Allows aborting an in-progress test file.
+  * `timeout` {number} A number of milliseconds the test file will fail after.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `Infinity`.
+* Returns: {Promise} Resolved with `undefined` once all test files complete.
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js
index d28311f3ab5dcd..f95921adf151bd 100644
--- a/lib/internal/main/test_runner.js
+++ b/lib/internal/main/test_runner.js
@@ -1,147 +1,11 @@
 'use strict';
-const {
-  ArrayFrom,
-  ArrayPrototypeFilter,
-  ArrayPrototypeIncludes,
-  ArrayPrototypeJoin,
-  ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSort,
-  SafePromiseAll,
-  SafeSet,
-} = primordials;
 const {
   prepareMainThreadExecution,
   markBootstrapComplete
 } = require('internal/process/pre_execution');
-const { spawn } = require('child_process');
-const { readdirSync, statSync } = require('fs');
-const console = require('internal/console/global');
-const {
-  codes: {
-    ERR_TEST_FAILURE,
-  },
-} = require('internal/errors');
-const { test } = require('internal/test_runner/harness');
-const { kSubtestsFailed } = require('internal/test_runner/test');
-const {
-  isSupportedFileType,
-  doesPathMatchFilter,
-} = require('internal/test_runner/utils');
-const { basename, join, resolve } = require('path');
-const { once } = require('events');
-const kFilterArgs = ['--test'];
+const { runFiles } = require('internal/test_runner/runner');
 
 prepareMainThreadExecution(false);
 markBootstrapComplete();
 
-// TODO(cjihrig): Replace this with recursive readdir once it lands.
-function processPath(path, testFiles, options) {
-  const stats = statSync(path);
-
-  if (stats.isFile()) {
-    if (options.userSupplied ||
-        (options.underTestDir && isSupportedFileType(path)) ||
-        doesPathMatchFilter(path)) {
-      testFiles.add(path);
-    }
-  } else if (stats.isDirectory()) {
-    const name = basename(path);
-
-    if (!options.userSupplied && name === 'node_modules') {
-      return;
-    }
-
-    // 'test' directories get special treatment. Recursively add all .js,
-    // .cjs, and .mjs files in the 'test' directory.
-    const isTestDir = name === 'test';
-    const { underTestDir } = options;
-    const entries = readdirSync(path);
-
-    if (isTestDir) {
-      options.underTestDir = true;
-    }
-
-    options.userSupplied = false;
-
-    for (let i = 0; i < entries.length; i++) {
-      processPath(join(path, entries[i]), testFiles, options);
-    }
-
-    options.underTestDir = underTestDir;
-  }
-}
-
-function createTestFileList() {
-  const cwd = process.cwd();
-  const hasUserSuppliedPaths = process.argv.length > 1;
-  const testPaths = hasUserSuppliedPaths ?
-    ArrayPrototypeSlice(process.argv, 1) : [cwd];
-  const testFiles = new SafeSet();
-
-  try {
-    for (let i = 0; i < testPaths.length; i++) {
-      const absolutePath = resolve(testPaths[i]);
-
-      processPath(absolutePath, testFiles, { userSupplied: true });
-    }
-  } catch (err) {
-    if (err?.code === 'ENOENT') {
-      console.error(`Could not find '${err.path}'`);
-      process.exit(1);
-    }
-
-    throw err;
-  }
-
-  return ArrayPrototypeSort(ArrayFrom(testFiles));
-}
-
-function filterExecArgv(arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg);
-}
-
-function runTestFile(path) {
-  return test(path, async (t) => {
-    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
-    ArrayPrototypePush(args, path);
-
-    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
-    let err;
-
-    child.on('error', (error) => {
-      err = error;
-    });
-
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      child.stdout.toArray({ signal: t.signal }),
-      child.stderr.toArray({ signal: t.signal }),
-    ]);
-
-    if (code !== 0 || signal !== null) {
-      if (!err) {
-        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
-        err.exitCode = code;
-        err.signal = signal;
-        err.stdout = ArrayPrototypeJoin(stdout, '');
-        err.stderr = ArrayPrototypeJoin(stderr, '');
-        // The stack will not be useful since the failures came from tests
-        // in a child process.
-        err.stack = undefined;
-      }
-
-      throw err;
-    }
-  });
-}
-
-(async function main() {
-  const testFiles = createTestFileList();
-
-  for (let i = 0; i < testFiles.length; i++) {
-    runTestFile(testFiles[i]);
-  }
-})();
+runFiles();
diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js
index 360fe141fe70a1..1fc5e06212c4b9 100644
--- a/lib/internal/test_runner/harness.js
+++ b/lib/internal/test_runner/harness.js
@@ -3,6 +3,7 @@ const {
   ArrayPrototypeForEach,
   FunctionPrototypeBind,
   SafeMap,
+  SafeWeakMap,
 } = primordials;
 const {
   createHook,
@@ -19,7 +20,11 @@ const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runne
 const isTestRunner = getOptionValue('--test');
 const testResources = new SafeMap();
 const root = new Test({ __proto__: null, name: '<root>' });
-let wasRootSetup = false;
+const wasRootSetup = new SafeWeakMap();
+
+function createTestTree(options = {}) {
+  return setup(new Test({ __proto__: null, ...options, name: '<root>' }));
+}
 
 function createProcessEventHandler(eventName, rootTest) {
   return (err) => {
@@ -48,7 +53,7 @@ function createProcessEventHandler(eventName, rootTest) {
 }
 
 function setup(root) {
-  if (wasRootSetup) {
+  if (wasRootSetup.has(root)) {
     return root;
   }
   const hook = createHook({
@@ -146,7 +151,7 @@ function setup(root) {
   root.reporter.pipe(process.stdout);
   root.reporter.version();
 
-  wasRootSetup = true;
+  wasRootSetup.set(root, true);
   return root;
 }
 
@@ -184,6 +189,7 @@ function hook(hook) {
 }
 
 module.exports = {
+  createTestTree,
   test: FunctionPrototypeBind(test, root),
   describe: runInParentContext(Suite),
   it: runInParentContext(ItTest),
diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js
new file mode 100644
index 00000000000000..7df9012d6228ac
--- /dev/null
+++ b/lib/internal/test_runner/runner.js
@@ -0,0 +1,154 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypeJoin,
+  ArrayPrototypeMap,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  SafePromiseAll,
+  SafeSet,
+} = primordials;
+
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const { validateArray } = require('internal/validators');
+const { kEmptyObject } = require('internal/util');
+const { createTestTree } = require('internal/test_runner/harness');
+const { kSubtestsFailed, Test } = require('internal/test_runner/test');
+const {
+  isSupportedFileType,
+  doesPathMatchFilter,
+} = require('internal/test_runner/utils');
+const { basename, join, resolve } = require('path');
+const { once } = require('events');
+
+const kFilterArgs = ['--test'];
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, options) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (options.userSupplied ||
+        (options.underTestDir && isSupportedFileType(path)) ||
+        doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path);
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return;
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test';
+    const { underTestDir } = options;
+    const entries = readdirSync(path);
+
+    if (isTestDir) {
+      options.underTestDir = true;
+    }
+
+    options.userSupplied = false;
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options);
+    }
+
+    options.underTestDir = underTestDir;
+  }
+}
+
+function createTestFileList() {
+  const cwd = process.cwd();
+  const hasUserSuppliedPaths = process.argv.length > 1;
+  const testPaths = hasUserSuppliedPaths ?
+    ArrayPrototypeSlice(process.argv, 1) : [cwd];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, { userSupplied: true });
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function filterExecArgv(arg) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+}
+
+function runTestFile(path, root) {
+  const subtest = root.createSubtest(Test, path, async (t) => {
+    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
+    ArrayPrototypePush(args, path);
+
+    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
+    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+    // instead of just displaying it all if the child fails.
+    let err;
+
+    child.on('error', (error) => {
+      err = error;
+    });
+
+    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
+      once(child, 'exit', { signal: t.signal }),
+      child.stdout.toArray({ signal: t.signal }),
+      child.stderr.toArray({ signal: t.signal }),
+    ]);
+
+    if (code !== 0 || signal !== null) {
+      if (!err) {
+        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
+        err.exitCode = code;
+        err.signal = signal;
+        err.stdout = ArrayPrototypeJoin(stdout, '');
+        err.stderr = ArrayPrototypeJoin(stderr, '');
+        // The stack will not be useful since the failures came from tests
+        // in a child process.
+        err.stack = undefined;
+      }
+
+      throw err;
+    }
+  });
+  return subtest.start();
+}
+
+async function runFiles(options) {
+  if (options === null || typeof options !== 'object') {
+    options = kEmptyObject;
+  }
+  const { concurrency, timeout, signal, files } = options;
+  const root = createTestTree({ concurrency, timeout, signal });
+
+  if (files !== undefined) {
+    validateArray(options, 'options.files');
+  }
+
+  await ArrayPrototypeMap(files ?? createTestFileList(), (path) => runTestFile(path, root));
+}
+
+module.exports = { runFiles };
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 3ebb4bc0a2dff9..82ccd280f14f71 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -176,7 +176,7 @@ class Test extends AsyncResource {
 
       case 'boolean':
         if (concurrency) {
-          this.concurrency = isTestRunner ? MathMax(cpus().length - 1, 1) : Infinity;
+          this.concurrency = parent === null ? MathMax(cpus().length - 1, 1) : Infinity;
         } else {
           this.concurrency = 1;
         }
diff --git a/lib/test.js b/lib/test.js
index a365eef6d45f91..ee53a12fd8ddd0 100644
--- a/lib/test.js
+++ b/lib/test.js
@@ -1,14 +1,10 @@
 'use strict';
+const { ObjectAssign } = primordials;
 const { test, describe, it, before, after, beforeEach, afterEach } = require('internal/test_runner/harness');
+const { runFiles } = require('internal/test_runner/runner');
 const { emitExperimentalWarning } = require('internal/util');
 
 emitExperimentalWarning('The test runner');
 
 module.exports = test;
-module.exports.test = test;
-module.exports.describe = describe;
-module.exports.it = it;
-module.exports.before = before;
-module.exports.after = after;
-module.exports.beforeEach = beforeEach;
-module.exports.afterEach = afterEach;
+ObjectAssign(module.exports, { test, describe, it, before, after, beforeEach, afterEach, runFiles });