Skip to content

Commit

Permalink
test_runner: support programmatically running --test
Browse files Browse the repository at this point in the history
PR-URL: #44241
Backport-PR-URL: #44873
Fixes: #44023
Fixes: #43675
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
  • Loading branch information
MoLow authored and richardlau committed Dec 7, 2022
1 parent 5a776d4 commit 37e9092
Show file tree
Hide file tree
Showing 10 changed files with 424 additions and 230 deletions.
74 changes: 74 additions & 0 deletions doc/api/test.md
Expand Up @@ -316,6 +316,35 @@ Otherwise, the test is considered to be a failure. Test files must be
executable by Node.js, but are not required to use the `node:test` module
internally.

## `run([options])`

<!-- YAML
added: REPLACEME
-->

* `options` {Object} Configuration options for running tests. The following
properties are supported:
* `concurrency` {number|boolean} If a number is provided,
then that many files would run in parallel.
If truthy, it would run (number of cpu cores - 1)
files in parallel.
If falsy, it would only run one file at a time.
If unspecified, subtests inherit this value from their parent.
**Default:** `true`.
* `files`: {Array} An array containing the list of files to run.
**Default** matching files from [test runner execution model][].
* `signal` {AbortSignal} Allows aborting an in-progress test execution.
* `timeout` {number} A number of milliseconds the test execution will
fail after.
If unspecified, subtests inherit this value from their parent.
**Default:** `Infinity`.
* Returns: {TapStream}

```js
run({ files: [path.resolve('./tests/test.js')] })
.pipe(process.stdout);
```

## `test([name][, options][, fn])`

<!-- YAML
Expand Down Expand Up @@ -560,6 +589,47 @@ describe('tests', async () => {
});
```

## Class: `TapStream`

<!-- YAML
added: REPLACEME
-->

* Extends {ReadableStream}

A successful call to [`run()`][] method will return a new {TapStream}
object, streaming a [TAP][] output
`TapStream` will emit events, in the order of the tests definition

### Event: `'test:diagnostic'`

* `message` {string} The diagnostic message.

Emitted when [`context.diagnostic`][] is called.

### Event: `'test:fail'`

* `data` {Object}
* `duration` {number} The test duration.
* `error` {Error} The failure casing test to fail.
* `name` {string} The test name.
* `testNumber` {number} The ordinal number of the test.
* `todo` {string|undefined} Present if [`context.todo`][] is called
* `skip` {string|undefined} Present if [`context.skip`][] is called

Emitted when a test fails.

### Event: `'test:pass'`

* `data` {Object}
* `duration` {number} The test duration.
* `name` {string} The test name.
* `testNumber` {number} The ordinal number of the test.
* `todo` {string|undefined} Present if [`context.todo`][] is called
* `skip` {string|undefined} Present if [`context.skip`][] is called

Emitted when a test passes.

## Class: `TestContext`

<!-- YAML
Expand Down Expand Up @@ -825,6 +895,10 @@ added: v16.17.0
[`--test`]: cli.md#--test
[`SuiteContext`]: #class-suitecontext
[`TestContext`]: #class-testcontext
[`context.diagnostic`]: #contextdiagnosticmessage
[`context.skip`]: #contextskipmessage
[`context.todo`]: #contexttodomessage
[`run()`]: #runoptions
[`test()`]: #testname-options-fn
[describe options]: #describename-options-fn
[it options]: #testname-options-fn
Expand Down
144 changes: 6 additions & 138 deletions lib/internal/main/test_runner.js
@@ -1,146 +1,14 @@
'use strict';
const {
ArrayFrom,
ArrayPrototypeFilter,
ArrayPrototypeIncludes,
ArrayPrototypeJoin,
ArrayPrototypePush,
ArrayPrototypeSlice,
ArrayPrototypeSort,
SafePromiseAll,
SafeSet,
} = primordials;
const {
prepareMainThreadExecution,
} = require('internal/bootstrap/pre_execution');
const { spawn } = require('child_process');
const { readdirSync, statSync } = require('fs');
const console = require('internal/console/global');
const {
codes: {
ERR_TEST_FAILURE,
},
} = require('internal/errors');
const { test } = require('internal/test_runner/harness');
const { kSubtestsFailed } = require('internal/test_runner/test');
const {
isSupportedFileType,
doesPathMatchFilter,
} = require('internal/test_runner/utils');
const { basename, join, resolve } = require('path');
const { once } = require('events');
const kFilterArgs = ['--test'];
const { run } = require('internal/test_runner/runner');

prepareMainThreadExecution(false);
markBootstrapComplete();

// TODO(cjihrig): Replace this with recursive readdir once it lands.
function processPath(path, testFiles, options) {
const stats = statSync(path);

if (stats.isFile()) {
if (options.userSupplied ||
(options.underTestDir && isSupportedFileType(path)) ||
doesPathMatchFilter(path)) {
testFiles.add(path);
}
} else if (stats.isDirectory()) {
const name = basename(path);

if (!options.userSupplied && name === 'node_modules') {
return;
}

// 'test' directories get special treatment. Recursively add all .js,
// .cjs, and .mjs files in the 'test' directory.
const isTestDir = name === 'test';
const { underTestDir } = options;
const entries = readdirSync(path);

if (isTestDir) {
options.underTestDir = true;
}

options.userSupplied = false;

for (let i = 0; i < entries.length; i++) {
processPath(join(path, entries[i]), testFiles, options);
}

options.underTestDir = underTestDir;
}
}

function createTestFileList() {
const cwd = process.cwd();
const hasUserSuppliedPaths = process.argv.length > 1;
const testPaths = hasUserSuppliedPaths ?
ArrayPrototypeSlice(process.argv, 1) : [cwd];
const testFiles = new SafeSet();

try {
for (let i = 0; i < testPaths.length; i++) {
const absolutePath = resolve(testPaths[i]);

processPath(absolutePath, testFiles, { userSupplied: true });
}
} catch (err) {
if (err?.code === 'ENOENT') {
console.error(`Could not find '${err.path}'`);
process.exit(1);
}

throw err;
}

return ArrayPrototypeSort(ArrayFrom(testFiles));
}

function filterExecArgv(arg) {
return !ArrayPrototypeIncludes(kFilterArgs, arg);
}

function runTestFile(path) {
return test(path, async (t) => {
const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
ArrayPrototypePush(args, path);

const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
// instead of just displaying it all if the child fails.
let err;

child.on('error', (error) => {
err = error;
});

const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
once(child, 'exit', { signal: t.signal }),
child.stdout.toArray({ signal: t.signal }),
child.stderr.toArray({ signal: t.signal }),
]);

if (code !== 0 || signal !== null) {
if (!err) {
err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
err.exitCode = code;
err.signal = signal;
err.stdout = ArrayPrototypeJoin(stdout, '');
err.stderr = ArrayPrototypeJoin(stderr, '');
// The stack will not be useful since the failures came from tests
// in a child process.
err.stack = undefined;
}

throw err;
}
});
}

(async function main() {
const testFiles = createTestFileList();

for (let i = 0; i < testFiles.length; i++) {
runTestFile(testFiles[i]);
}
})();
const tapStream = run();
tapStream.pipe(process.stdout);
tapStream.once('test:fail', () => {
process.exitCode = 1;
});

0 comments on commit 37e9092

Please sign in to comment.