Skip to content

Commit 37e9092

Browse files
MoLowrichardlau
MoLow
authored and
richardlau
committedDec 7, 2022
test_runner: support programmatically running --test
PR-URL: #44241 Backport-PR-URL: #44873 Fixes: #44023 Fixes: #43675 Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com> Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
·
v16.20.2v16.19.0
1 parent 5a776d4 commit 37e9092

File tree

10 files changed

+424
-230
lines changed

10 files changed

+424
-230
lines changed
 

‎doc/api/test.md

+74
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,35 @@ Otherwise, the test is considered to be a failure. Test files must be
316316
executable by Node.js, but are not required to use the `node:test` module
317317
internally.
318318

319+
## `run([options])`
320+
321+
<!-- YAML
322+
added: REPLACEME
323+
-->
324+
325+
* `options` {Object} Configuration options for running tests. The following
326+
properties are supported:
327+
* `concurrency` {number|boolean} If a number is provided,
328+
then that many files would run in parallel.
329+
If truthy, it would run (number of cpu cores - 1)
330+
files in parallel.
331+
If falsy, it would only run one file at a time.
332+
If unspecified, subtests inherit this value from their parent.
333+
**Default:** `true`.
334+
* `files`: {Array} An array containing the list of files to run.
335+
**Default** matching files from [test runner execution model][].
336+
* `signal` {AbortSignal} Allows aborting an in-progress test execution.
337+
* `timeout` {number} A number of milliseconds the test execution will
338+
fail after.
339+
If unspecified, subtests inherit this value from their parent.
340+
**Default:** `Infinity`.
341+
* Returns: {TapStream}
342+
343+
```js
344+
run({ files: [path.resolve('./tests/test.js')] })
345+
.pipe(process.stdout);
346+
```
347+
319348
## `test([name][, options][, fn])`
320349

321350
<!-- YAML
@@ -560,6 +589,47 @@ describe('tests', async () => {
560589
});
561590
```
562591

592+
## Class: `TapStream`
593+
594+
<!-- YAML
595+
added: REPLACEME
596+
-->
597+
598+
* Extends {ReadableStream}
599+
600+
A successful call to [`run()`][] method will return a new {TapStream}
601+
object, streaming a [TAP][] output
602+
`TapStream` will emit events, in the order of the tests definition
603+
604+
### Event: `'test:diagnostic'`
605+
606+
* `message` {string} The diagnostic message.
607+
608+
Emitted when [`context.diagnostic`][] is called.
609+
610+
### Event: `'test:fail'`
611+
612+
* `data` {Object}
613+
* `duration` {number} The test duration.
614+
* `error` {Error} The failure casing test to fail.
615+
* `name` {string} The test name.
616+
* `testNumber` {number} The ordinal number of the test.
617+
* `todo` {string|undefined} Present if [`context.todo`][] is called
618+
* `skip` {string|undefined} Present if [`context.skip`][] is called
619+
620+
Emitted when a test fails.
621+
622+
### Event: `'test:pass'`
623+
624+
* `data` {Object}
625+
* `duration` {number} The test duration.
626+
* `name` {string} The test name.
627+
* `testNumber` {number} The ordinal number of the test.
628+
* `todo` {string|undefined} Present if [`context.todo`][] is called
629+
* `skip` {string|undefined} Present if [`context.skip`][] is called
630+
631+
Emitted when a test passes.
632+
563633
## Class: `TestContext`
564634

565635
<!-- YAML
@@ -825,6 +895,10 @@ added: v16.17.0
825895
[`--test`]: cli.md#--test
826896
[`SuiteContext`]: #class-suitecontext
827897
[`TestContext`]: #class-testcontext
898+
[`context.diagnostic`]: #contextdiagnosticmessage
899+
[`context.skip`]: #contextskipmessage
900+
[`context.todo`]: #contexttodomessage
901+
[`run()`]: #runoptions
828902
[`test()`]: #testname-options-fn
829903
[describe options]: #describename-options-fn
830904
[it options]: #testname-options-fn

‎lib/internal/main/test_runner.js

+6-138
Original file line numberDiff line numberDiff line change
@@ -1,146 +1,14 @@
11
'use strict';
2-
const {
3-
ArrayFrom,
4-
ArrayPrototypeFilter,
5-
ArrayPrototypeIncludes,
6-
ArrayPrototypeJoin,
7-
ArrayPrototypePush,
8-
ArrayPrototypeSlice,
9-
ArrayPrototypeSort,
10-
SafePromiseAll,
11-
SafeSet,
12-
} = primordials;
132
const {
143
prepareMainThreadExecution,
154
} = require('internal/bootstrap/pre_execution');
16-
const { spawn } = require('child_process');
17-
const { readdirSync, statSync } = require('fs');
18-
const console = require('internal/console/global');
19-
const {
20-
codes: {
21-
ERR_TEST_FAILURE,
22-
},
23-
} = require('internal/errors');
24-
const { test } = require('internal/test_runner/harness');
25-
const { kSubtestsFailed } = require('internal/test_runner/test');
26-
const {
27-
isSupportedFileType,
28-
doesPathMatchFilter,
29-
} = require('internal/test_runner/utils');
30-
const { basename, join, resolve } = require('path');
31-
const { once } = require('events');
32-
const kFilterArgs = ['--test'];
5+
const { run } = require('internal/test_runner/runner');
336

347
prepareMainThreadExecution(false);
358
markBootstrapComplete();
369

37-
// TODO(cjihrig): Replace this with recursive readdir once it lands.
38-
function processPath(path, testFiles, options) {
39-
const stats = statSync(path);
40-
41-
if (stats.isFile()) {
42-
if (options.userSupplied ||
43-
(options.underTestDir && isSupportedFileType(path)) ||
44-
doesPathMatchFilter(path)) {
45-
testFiles.add(path);
46-
}
47-
} else if (stats.isDirectory()) {
48-
const name = basename(path);
49-
50-
if (!options.userSupplied && name === 'node_modules') {
51-
return;
52-
}
53-
54-
// 'test' directories get special treatment. Recursively add all .js,
55-
// .cjs, and .mjs files in the 'test' directory.
56-
const isTestDir = name === 'test';
57-
const { underTestDir } = options;
58-
const entries = readdirSync(path);
59-
60-
if (isTestDir) {
61-
options.underTestDir = true;
62-
}
63-
64-
options.userSupplied = false;
65-
66-
for (let i = 0; i < entries.length; i++) {
67-
processPath(join(path, entries[i]), testFiles, options);
68-
}
69-
70-
options.underTestDir = underTestDir;
71-
}
72-
}
73-
74-
function createTestFileList() {
75-
const cwd = process.cwd();
76-
const hasUserSuppliedPaths = process.argv.length > 1;
77-
const testPaths = hasUserSuppliedPaths ?
78-
ArrayPrototypeSlice(process.argv, 1) : [cwd];
79-
const testFiles = new SafeSet();
80-
81-
try {
82-
for (let i = 0; i < testPaths.length; i++) {
83-
const absolutePath = resolve(testPaths[i]);
84-
85-
processPath(absolutePath, testFiles, { userSupplied: true });
86-
}
87-
} catch (err) {
88-
if (err?.code === 'ENOENT') {
89-
console.error(`Could not find '${err.path}'`);
90-
process.exit(1);
91-
}
92-
93-
throw err;
94-
}
95-
96-
return ArrayPrototypeSort(ArrayFrom(testFiles));
97-
}
98-
99-
function filterExecArgv(arg) {
100-
return !ArrayPrototypeIncludes(kFilterArgs, arg);
101-
}
102-
103-
function runTestFile(path) {
104-
return test(path, async (t) => {
105-
const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
106-
ArrayPrototypePush(args, path);
107-
108-
const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
109-
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
110-
// instead of just displaying it all if the child fails.
111-
let err;
112-
113-
child.on('error', (error) => {
114-
err = error;
115-
});
116-
117-
const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
118-
once(child, 'exit', { signal: t.signal }),
119-
child.stdout.toArray({ signal: t.signal }),
120-
child.stderr.toArray({ signal: t.signal }),
121-
]);
122-
123-
if (code !== 0 || signal !== null) {
124-
if (!err) {
125-
err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
126-
err.exitCode = code;
127-
err.signal = signal;
128-
err.stdout = ArrayPrototypeJoin(stdout, '');
129-
err.stderr = ArrayPrototypeJoin(stderr, '');
130-
// The stack will not be useful since the failures came from tests
131-
// in a child process.
132-
err.stack = undefined;
133-
}
134-
135-
throw err;
136-
}
137-
});
138-
}
139-
140-
(async function main() {
141-
const testFiles = createTestFileList();
142-
143-
for (let i = 0; i < testFiles.length; i++) {
144-
runTestFile(testFiles[i]);
145-
}
146-
})();
10+
const tapStream = run();
11+
tapStream.pipe(process.stdout);
12+
tapStream.once('test:fail', () => {
13+
process.exitCode = 1;
14+
});

‎lib/internal/test_runner/harness.js

+32-57
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
'use strict';
22
const {
33
ArrayPrototypeForEach,
4-
FunctionPrototypeBind,
54
SafeMap,
5+
SafeWeakSet,
66
} = primordials;
77
const {
88
createHook,
@@ -13,13 +13,18 @@ const {
1313
ERR_TEST_FAILURE,
1414
},
1515
} = require('internal/errors');
16+
const { kEmptyObject } = require('internal/util');
1617
const { getOptionValue } = require('internal/options');
1718
const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test');
19+
const { bigint: hrtime } = process.hrtime;
1820

19-
const isTestRunner = getOptionValue('--test');
21+
const isTestRunnerCli = getOptionValue('--test');
2022
const testResources = new SafeMap();
21-
const root = new Test({ __proto__: null, name: '<root>' });
22-
let wasRootSetup = false;
23+
const wasRootSetup = new SafeWeakSet();
24+
25+
function createTestTree(options = kEmptyObject) {
26+
return setup(new Test({ __proto__: null, ...options, name: '<root>' }));
27+
}
2328

2429
function createProcessEventHandler(eventName, rootTest) {
2530
return (err) => {
@@ -48,7 +53,7 @@ function createProcessEventHandler(eventName, rootTest) {
4853
}
4954

5055
function setup(root) {
51-
if (wasRootSetup) {
56+
if (wasRootSetup.has(root)) {
5257
return root;
5358
}
5459
const hook = createHook({
@@ -81,52 +86,9 @@ function setup(root) {
8186
'Promise resolution is still pending but the event loop has already resolved',
8287
kCancelledByParent));
8388

84-
let passCount = 0;
85-
let failCount = 0;
86-
let skipCount = 0;
87-
let todoCount = 0;
88-
let cancelledCount = 0;
89-
90-
for (let i = 0; i < root.subtests.length; i++) {
91-
const test = root.subtests[i];
92-
93-
// Check SKIP and TODO tests first, as those should not be counted as
94-
// failures.
95-
if (test.skipped) {
96-
skipCount++;
97-
} else if (test.isTodo) {
98-
todoCount++;
99-
} else if (test.cancelled) {
100-
cancelledCount++;
101-
} else if (!test.passed) {
102-
failCount++;
103-
} else {
104-
passCount++;
105-
}
106-
}
107-
108-
root.reporter.plan(root.indent, root.subtests.length);
109-
110-
for (let i = 0; i < root.diagnostics.length; i++) {
111-
root.reporter.diagnostic(root.indent, root.diagnostics[i]);
112-
}
113-
114-
root.reporter.diagnostic(root.indent, `tests ${root.subtests.length}`);
115-
root.reporter.diagnostic(root.indent, `pass ${passCount}`);
116-
root.reporter.diagnostic(root.indent, `fail ${failCount}`);
117-
root.reporter.diagnostic(root.indent, `cancelled ${cancelledCount}`);
118-
root.reporter.diagnostic(root.indent, `skipped ${skipCount}`);
119-
root.reporter.diagnostic(root.indent, `todo ${todoCount}`);
120-
root.reporter.diagnostic(root.indent, `duration_ms ${process.uptime()}`);
121-
122-
root.reporter.push(null);
12389
hook.disable();
12490
process.removeListener('unhandledRejection', rejectionHandler);
12591
process.removeListener('uncaughtException', exceptionHandler);
126-
127-
if (failCount > 0 || cancelledCount > 0) {
128-
process.exitCode = 1;
129-
}
13092
};
13193

13294
const terminationHandler = () => {
@@ -137,29 +99,41 @@ function setup(root) {
13799
process.on('uncaughtException', exceptionHandler);
138100
process.on('unhandledRejection', rejectionHandler);
139101
process.on('beforeExit', exitHandler);
140-
// TODO(MoLow): Make it configurable to hook when isTestRunner === false.
141-
if (isTestRunner) {
102+
// TODO(MoLow): Make it configurable to hook when isTestRunnerCli === false.
103+
if (isTestRunnerCli) {
142104
process.on('SIGINT', terminationHandler);
143105
process.on('SIGTERM', terminationHandler);
144106
}
145107

146-
root.reporter.pipe(process.stdout);
108+
root.startTime = hrtime();
147109
root.reporter.version();
148110

149-
wasRootSetup = true;
111+
wasRootSetup.add(root);
150112
return root;
151113
}
152114

115+
let globalRoot;
116+
function getGlobalRoot() {
117+
if (!globalRoot) {
118+
globalRoot = createTestTree();
119+
globalRoot.reporter.pipe(process.stdout);
120+
globalRoot.reporter.once('test:fail', () => {
121+
process.exitCode = 1;
122+
});
123+
}
124+
return globalRoot;
125+
}
126+
153127
function test(name, options, fn) {
154-
const subtest = setup(root).createSubtest(Test, name, options, fn);
128+
const subtest = getGlobalRoot().createSubtest(Test, name, options, fn);
155129
return subtest.start();
156130
}
157131

158132
function runInParentContext(Factory) {
159133
function run(name, options, fn, overrides) {
160-
const parent = testResources.get(executionAsyncId()) || setup(root);
134+
const parent = testResources.get(executionAsyncId()) || getGlobalRoot();
161135
const subtest = parent.createSubtest(Factory, name, options, fn, overrides);
162-
if (parent === root) {
136+
if (parent === getGlobalRoot()) {
163137
subtest.start();
164138
}
165139
}
@@ -178,13 +152,14 @@ function runInParentContext(Factory) {
178152

179153
function hook(hook) {
180154
return (fn, options) => {
181-
const parent = testResources.get(executionAsyncId()) || setup(root);
155+
const parent = testResources.get(executionAsyncId()) || getGlobalRoot();
182156
parent.createHook(hook, fn, options);
183157
};
184158
}
185159

186160
module.exports = {
187-
test: FunctionPrototypeBind(test, root),
161+
createTestTree,
162+
test,
188163
describe: runInParentContext(Suite),
189164
it: runInParentContext(ItTest),
190165
before: hook('before'),

‎lib/internal/test_runner/runner.js

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
'use strict';
2+
const {
3+
ArrayFrom,
4+
ArrayPrototypeConcat,
5+
ArrayPrototypeFilter,
6+
ArrayPrototypeIncludes,
7+
ArrayPrototypeJoin,
8+
ArrayPrototypeSlice,
9+
ArrayPrototypeSort,
10+
ObjectAssign,
11+
PromisePrototypeThen,
12+
SafePromiseAll,
13+
SafeSet,
14+
} = primordials;
15+
16+
const { spawn } = require('child_process');
17+
const { readdirSync, statSync } = require('fs');
18+
const console = require('internal/console/global');
19+
const {
20+
codes: {
21+
ERR_TEST_FAILURE,
22+
},
23+
} = require('internal/errors');
24+
const { validateArray } = require('internal/validators');
25+
const { kEmptyObject } = require('internal/util');
26+
const { createTestTree } = require('internal/test_runner/harness');
27+
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
28+
const {
29+
isSupportedFileType,
30+
doesPathMatchFilter,
31+
} = require('internal/test_runner/utils');
32+
const { basename, join, resolve } = require('path');
33+
const { once } = require('events');
34+
35+
const kFilterArgs = ['--test'];
36+
37+
// TODO(cjihrig): Replace this with recursive readdir once it lands.
38+
function processPath(path, testFiles, options) {
39+
const stats = statSync(path);
40+
41+
if (stats.isFile()) {
42+
if (options.userSupplied ||
43+
(options.underTestDir && isSupportedFileType(path)) ||
44+
doesPathMatchFilter(path)) {
45+
testFiles.add(path);
46+
}
47+
} else if (stats.isDirectory()) {
48+
const name = basename(path);
49+
50+
if (!options.userSupplied && name === 'node_modules') {
51+
return;
52+
}
53+
54+
// 'test' directories get special treatment. Recursively add all .js,
55+
// .cjs, and .mjs files in the 'test' directory.
56+
const isTestDir = name === 'test';
57+
const { underTestDir } = options;
58+
const entries = readdirSync(path);
59+
60+
if (isTestDir) {
61+
options.underTestDir = true;
62+
}
63+
64+
options.userSupplied = false;
65+
66+
for (let i = 0; i < entries.length; i++) {
67+
processPath(join(path, entries[i]), testFiles, options);
68+
}
69+
70+
options.underTestDir = underTestDir;
71+
}
72+
}
73+
74+
function createTestFileList() {
75+
const cwd = process.cwd();
76+
const hasUserSuppliedPaths = process.argv.length > 1;
77+
const testPaths = hasUserSuppliedPaths ?
78+
ArrayPrototypeSlice(process.argv, 1) : [cwd];
79+
const testFiles = new SafeSet();
80+
81+
try {
82+
for (let i = 0; i < testPaths.length; i++) {
83+
const absolutePath = resolve(testPaths[i]);
84+
85+
processPath(absolutePath, testFiles, { userSupplied: true });
86+
}
87+
} catch (err) {
88+
if (err?.code === 'ENOENT') {
89+
console.error(`Could not find '${err.path}'`);
90+
process.exit(1);
91+
}
92+
93+
throw err;
94+
}
95+
96+
return ArrayPrototypeSort(ArrayFrom(testFiles));
97+
}
98+
99+
function filterExecArgv(arg) {
100+
return !ArrayPrototypeIncludes(kFilterArgs, arg);
101+
}
102+
103+
function runTestFile(path, root) {
104+
const subtest = root.createSubtest(Test, path, async (t) => {
105+
const args = ArrayPrototypeConcat(
106+
ArrayPrototypeFilter(process.execArgv, filterExecArgv),
107+
path);
108+
109+
const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
110+
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
111+
// instead of just displaying it all if the child fails.
112+
let err;
113+
114+
child.on('error', (error) => {
115+
err = error;
116+
});
117+
118+
const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
119+
once(child, 'exit', { signal: t.signal }),
120+
child.stdout.toArray({ signal: t.signal }),
121+
child.stderr.toArray({ signal: t.signal }),
122+
]);
123+
124+
if (code !== 0 || signal !== null) {
125+
if (!err) {
126+
err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), {
127+
__proto__: null,
128+
exitCode: code,
129+
signal: signal,
130+
stdout: ArrayPrototypeJoin(stdout, ''),
131+
stderr: ArrayPrototypeJoin(stderr, ''),
132+
// The stack will not be useful since the failures came from tests
133+
// in a child process.
134+
stack: undefined,
135+
});
136+
}
137+
138+
throw err;
139+
}
140+
});
141+
return subtest.start();
142+
}
143+
144+
function run(options) {
145+
if (options === null || typeof options !== 'object') {
146+
options = kEmptyObject;
147+
}
148+
const { concurrency, timeout, signal, files } = options;
149+
150+
if (files != null) {
151+
validateArray(files, 'options.files');
152+
}
153+
154+
const root = createTestTree({ concurrency, timeout, signal });
155+
const testFiles = files ?? createTestFileList();
156+
157+
PromisePrototypeThen(SafePromiseAll(testFiles, (path) => runTestFile(path, root)),
158+
() => root.postRun());
159+
160+
return root.reporter;
161+
}
162+
163+
module.exports = { run };

‎lib/internal/test_runner/tap_stream.js

+20-14
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const {
33
ArrayPrototypeForEach,
44
ArrayPrototypeJoin,
5+
ArrayPrototypeMap,
56
ArrayPrototypePush,
67
ArrayPrototypeShift,
78
ObjectEntries,
@@ -11,7 +12,7 @@ const {
1112
} = primordials;
1213
const { inspectWithNoCustomRetry } = require('internal/errors');
1314
const Readable = require('internal/streams/readable');
14-
const { isError } = require('internal/util');
15+
const { isError, kEmptyObject } = require('internal/util');
1516
const kFrameStartRegExp = /^ {4}at /;
1617
const kLineBreakRegExp = /\n|\r\n/;
1718
const inspectOptions = { colors: false, breakLength: Infinity };
@@ -49,12 +50,16 @@ class TapStream extends Readable {
4950
this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`);
5051
}
5152

52-
fail(indent, testNumber, description, directive) {
53-
this.#test(indent, testNumber, 'not ok', description, directive);
53+
fail(indent, testNumber, name, duration, error, directive) {
54+
this.emit('test:fail', { __proto__: null, name, testNumber, duration, ...directive, error });
55+
this.#test(indent, testNumber, 'not ok', name, directive);
56+
this.#details(indent, duration, error);
5457
}
5558

56-
ok(indent, testNumber, description, directive) {
57-
this.#test(indent, testNumber, 'ok', description, directive);
59+
ok(indent, testNumber, name, duration, directive) {
60+
this.emit('test:pass', { __proto__: null, name, testNumber, duration, ...directive });
61+
this.#test(indent, testNumber, 'ok', name, directive);
62+
this.#details(indent, duration, null);
5863
}
5964

6065
plan(indent, count, explanation) {
@@ -64,18 +69,18 @@ class TapStream extends Readable {
6469
}
6570

6671
getSkip(reason) {
67-
return `SKIP${reason ? ` ${tapEscape(reason)}` : ''}`;
72+
return { __proto__: null, skip: reason };
6873
}
6974

7075
getTodo(reason) {
71-
return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`;
76+
return { __proto__: null, todo: reason };
7277
}
7378

7479
subtest(indent, name) {
7580
this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`);
7681
}
7782

78-
details(indent, duration, error) {
83+
#details(indent, duration, error) {
7984
let details = `${indent} ---\n`;
8085

8186
details += jsToYaml(indent, 'duration_ms', duration);
@@ -85,23 +90,24 @@ class TapStream extends Readable {
8590
}
8691

8792
diagnostic(indent, message) {
93+
this.emit('test:diagnostic', message);
8894
this.#tryPush(`${indent}# ${tapEscape(message)}\n`);
8995
}
9096

9197
version() {
9298
this.#tryPush('TAP version 13\n');
9399
}
94100

95-
#test(indent, testNumber, status, description, directive) {
101+
#test(indent, testNumber, status, name, directive = kEmptyObject) {
96102
let line = `${indent}${status} ${testNumber}`;
97103

98-
if (description) {
99-
line += ` ${tapEscape(description)}`;
104+
if (name) {
105+
line += ` ${tapEscape(`- ${name}`)}`;
100106
}
101107

102-
if (directive) {
103-
line += ` # ${directive}`;
104-
}
108+
line += ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(directive), ({ 0: key, 1: value }) => (
109+
` # ${key.toUpperCase()}${value ? ` ${tapEscape(value)}` : ''}`
110+
)), '');
105111

106112
line += '\n';
107113
this.#tryPush(line);

‎lib/internal/test_runner/test.js

+45-13
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ class Test extends AsyncResource {
180180

181181
case 'boolean':
182182
if (concurrency) {
183-
this.concurrency = isTestRunner ? MathMax(cpus().length - 1, 1) : Infinity;
183+
this.concurrency = parent === null ? MathMax(cpus().length - 1, 1) : Infinity;
184184
} else {
185185
this.concurrency = 1;
186186
}
@@ -516,7 +516,7 @@ class Test extends AsyncResource {
516516
}
517517

518518
postRun(pendingSubtestsError) {
519-
let failedSubtests = 0;
519+
const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
520520

521521
// If the test was failed before it even started, then the end time will
522522
// be earlier than the start time. Correct that here.
@@ -536,14 +536,28 @@ class Test extends AsyncResource {
536536
subtest.postRun(pendingSubtestsError);
537537
}
538538

539+
// Check SKIP and TODO tests first, as those should not be counted as
540+
// failures.
541+
if (subtest.skipped) {
542+
counters.skipped++;
543+
} else if (subtest.isTodo) {
544+
counters.todo++;
545+
} else if (subtest.cancelled) {
546+
counters.cancelled++;
547+
} else if (!subtest.passed) {
548+
counters.failed++;
549+
} else {
550+
counters.passed++;
551+
}
552+
539553
if (!subtest.passed) {
540-
failedSubtests++;
554+
counters.totalFailed++;
541555
}
542556
}
543557

544-
if (this.passed && failedSubtests > 0) {
545-
const subtestString = `subtest${failedSubtests > 1 ? 's' : ''}`;
546-
const msg = `${failedSubtests} ${subtestString} failed`;
558+
if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
559+
const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
560+
const msg = `${counters.totalFailed} ${subtestString} failed`;
547561

548562
this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
549563
}
@@ -555,6 +569,22 @@ class Test extends AsyncResource {
555569
this.parent.addReadySubtest(this);
556570
this.parent.processReadySubtestRange(false);
557571
this.parent.processPendingSubtests();
572+
} else if (!this.reported) {
573+
this.reported = true;
574+
this.reporter.plan(this.indent, this.subtests.length);
575+
576+
for (let i = 0; i < this.diagnostics.length; i++) {
577+
this.reporter.diagnostic(this.indent, this.diagnostics[i]);
578+
}
579+
580+
this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`);
581+
this.reporter.diagnostic(this.indent, `pass ${counters.passed}`);
582+
this.reporter.diagnostic(this.indent, `fail ${counters.failed}`);
583+
this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`);
584+
this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`);
585+
this.reporter.diagnostic(this.indent, `todo ${counters.todo}`);
586+
this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`);
587+
this.reporter.push(null);
558588
}
559589
}
560590

@@ -588,10 +618,12 @@ class Test extends AsyncResource {
588618
this.finished = true;
589619
}
590620

591-
report() {
621+
#duration() {
592622
// Duration is recorded in BigInt nanoseconds. Convert to seconds.
593-
const duration = Number(this.endTime - this.startTime) / 1_000_000_000;
594-
const message = `- ${this.name}`;
623+
return Number(this.endTime - this.startTime) / 1_000_000_000;
624+
}
625+
626+
report() {
595627
let directive;
596628

597629
if (this.skipped) {
@@ -601,13 +633,11 @@ class Test extends AsyncResource {
601633
}
602634

603635
if (this.passed) {
604-
this.reporter.ok(this.indent, this.testNumber, message, directive);
636+
this.reporter.ok(this.indent, this.testNumber, this.name, this.#duration(), directive);
605637
} else {
606-
this.reporter.fail(this.indent, this.testNumber, message, directive);
638+
this.reporter.fail(this.indent, this.testNumber, this.name, this.#duration(), this.error, directive);
607639
}
608640

609-
this.reporter.details(this.indent, duration, this.error);
610-
611641
for (let i = 0; i < this.diagnostics.length; i++) {
612642
this.reporter.diagnostic(this.indent, this.diagnostics[i]);
613643
}
@@ -630,6 +660,8 @@ class TestHook extends Test {
630660
getRunArgs() {
631661
return this.#args;
632662
}
663+
postRun() {
664+
}
633665
}
634666

635667
class ItTest extends Test {

‎lib/test.js

+12-7
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
'use strict';
2+
const { ObjectAssign } = primordials;
23
const { test, describe, it, before, after, beforeEach, afterEach } = require('internal/test_runner/harness');
4+
const { run } = require('internal/test_runner/runner');
35
const { emitExperimentalWarning } = require('internal/util');
46

57
emitExperimentalWarning('The test runner');
68

79
module.exports = test;
8-
module.exports.test = test;
9-
module.exports.describe = describe;
10-
module.exports.it = it;
11-
module.exports.before = before;
12-
module.exports.after = after;
13-
module.exports.beforeEach = beforeEach;
14-
module.exports.afterEach = afterEach;
10+
ObjectAssign(module.exports, {
11+
after,
12+
afterEach,
13+
before,
14+
beforeEach,
15+
describe,
16+
it,
17+
run,
18+
test,
19+
});

‎test/message/test_runner_no_tests.out

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
bound test
1+
test

‎test/parallel/test-runner-run.mjs

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import * as common from '../common/index.mjs';
2+
import * as fixtures from '../common/fixtures.mjs';
3+
import { join } from 'node:path';
4+
import { describe, it, run } from 'node:test';
5+
import assert from 'node:assert';
6+
7+
const testFixtures = fixtures.path('test-runner');
8+
9+
describe('require(\'node:test\').run', { concurrency: true }, () => {
10+
11+
it('should run with no tests', async () => {
12+
const stream = run({ files: [] });
13+
stream.setEncoding('utf8');
14+
stream.on('test:fail', common.mustNotCall());
15+
stream.on('test:pass', common.mustNotCall());
16+
// eslint-disable-next-line no-unused-vars
17+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
18+
});
19+
20+
it('should fail with non existing file', async () => {
21+
const stream = run({ files: ['a-random-file-that-does-not-exist.js'] });
22+
stream.on('test:fail', common.mustCall(1));
23+
stream.on('test:pass', common.mustNotCall());
24+
// eslint-disable-next-line no-unused-vars
25+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
26+
});
27+
28+
it('should succeed with a file', async () => {
29+
const stream = run({ files: [join(testFixtures, 'test/random.cjs')] });
30+
stream.on('test:fail', common.mustNotCall());
31+
stream.on('test:pass', common.mustCall(1));
32+
// eslint-disable-next-line no-unused-vars
33+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
34+
});
35+
36+
it('should run same file twice', async () => {
37+
const stream = run({ files: [join(testFixtures, 'test/random.cjs'), join(testFixtures, 'test/random.cjs')] });
38+
stream.on('test:fail', common.mustNotCall());
39+
stream.on('test:pass', common.mustCall(2));
40+
// eslint-disable-next-line no-unused-vars
41+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
42+
});
43+
44+
it('should run a failed test', async () => {
45+
const stream = run({ files: [testFixtures] });
46+
stream.on('test:fail', common.mustCall(1));
47+
stream.on('test:pass', common.mustNotCall());
48+
// eslint-disable-next-line no-unused-vars
49+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
50+
});
51+
52+
it('should support timeout', async () => {
53+
const stream = run({ timeout: 50, files: [
54+
fixtures.path('test-runner', 'never_ending_sync.js'),
55+
fixtures.path('test-runner', 'never_ending_async.js'),
56+
] });
57+
stream.on('test:fail', common.mustCall(2));
58+
stream.on('test:pass', common.mustNotCall());
59+
// eslint-disable-next-line no-unused-vars
60+
for await (const _ of stream); // TODO(MoLow): assert.snapshot
61+
});
62+
63+
it('should validate files', async () => {
64+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve([]), true, false]
65+
.forEach((files) => assert.throws(() => run({ files }), {
66+
code: 'ERR_INVALID_ARG_TYPE'
67+
}));
68+
});
69+
});

‎tools/doc/type-parser.mjs

+2
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,8 @@ const customTypesMap = {
201201
'Timeout': 'timers.html#class-timeout',
202202
'Timer': 'timers.html#timers',
203203

204+
'TapStream': 'test.html#class-tapstream',
205+
204206
'tls.SecureContext': 'tls.html#tlscreatesecurecontextoptions',
205207
'tls.Server': 'tls.html#class-tlsserver',
206208
'tls.TLSSocket': 'tls.html#class-tlstlssocket',

0 commit comments

Comments
 (0)
Please sign in to comment.