Skip to content

Commit

Permalink
test_runner: add reporters
Browse files Browse the repository at this point in the history
  • Loading branch information
@MoLow
MoLow committed Dec 2, 2022
1 parent 2cad517 commit 98598f0
Show file tree
Hide file tree
Showing 10 changed files with 186 additions and 65 deletions.
3 changes: 2 additions & 1 deletion lib/internal/main/test_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const {
const { getOptionValue } = require('internal/options');
const { isUsingInspector } = require('internal/util/inspector');
const { run } = require('internal/test_runner/runner');
const { getTestReporter } = require('internal/test_runner/utils');
const { exitCodes: { kGenericUserError } } = internalBinding('errors');

prepareMainThreadExecution(false);
Expand All @@ -21,7 +22,7 @@ if (isUsingInspector()) {
inspectPort = process.debugPort;
}

const tapStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') });
const tapStream = getTestReporter(run({ concurrency, inspectPort, watch: getOptionValue('--watch') }));
tapStream.pipe(process.stdout);
tapStream.once('test:fail', () => {
process.exitCode = kGenericUserError;
Expand Down
6 changes: 4 additions & 2 deletions lib/internal/test_runner/harness.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const { exitCodes: { kGenericUserError } } = internalBinding('errors');
const { kEmptyObject } = require('internal/util');
const { getOptionValue } = require('internal/options');
const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test');
const { getTestReporter } = require('internal/test_runner/utils');
const { bigint: hrtime } = process.hrtime;

const isTestRunnerCli = getOptionValue('--test');
Expand Down Expand Up @@ -119,8 +120,9 @@ let globalRoot;
function getGlobalRoot() {
if (!globalRoot) {
globalRoot = createTestTree();
globalRoot.reporter.pipe(process.stdout);
globalRoot.reporter.once('test:fail', () => {
const reporter = getTestReporter(globalRoot.reporter);
reporter.pipe(process.stdout);
reporter.once('test:fail', () => {
process.exitCode = kGenericUserError;
});
}
Expand Down
27 changes: 16 additions & 11 deletions lib/internal/test_runner/runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ const { validateArray, validateBoolean } = require('internal/validators');
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
const { kEmptyObject } = require('internal/util');
const { createTestTree } = require('internal/test_runner/harness');
const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test');
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
const { TapParser } = require('internal/test_runner/tap_parser');
const { kDefaultIndent } = require('internal/test_runner/tap_stream');
const { TokenKind } = require('internal/test_runner/tap_lexer');

const {
Expand All @@ -49,6 +50,7 @@ const {
} = internalBinding('errors');

const kFilterArgs = ['--test', '--watch'];
const kFilterArgValues = ['--test-reporter'];

// TODO(cjihrig): Replace this with recursive readdir once it lands.
function processPath(path, testFiles, options) {
Expand Down Expand Up @@ -112,8 +114,10 @@ function createTestFileList() {
return ArrayPrototypeSort(ArrayFrom(testFiles));
}

function filterExecArgv(arg) {
return !ArrayPrototypeIncludes(kFilterArgs, arg);
function filterExecArgv(arg, i, arr) {
return !ArrayPrototypeIncludes(kFilterArgs, arg) &&
ArrayPrototypeIncludes(kFilterArgValues, arg) &&
!ArrayPrototypeIncludes(kFilterArgValues, arr[i - 1]);
}

function getRunArgs({ path, inspectPort }) {
Expand All @@ -128,9 +132,10 @@ function getRunArgs({ path, inspectPort }) {
class FileTest extends Test {
#buffer = [];
#handleReportItem({ kind, node, nesting = 0 }) {
const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1);
nesting += 1;

const details = (diagnostic) => {
const indent = StringPrototypeRepeat(kDefaultIndent, nesting);
return (
diagnostic && {
__proto__: null,
Expand All @@ -149,11 +154,11 @@ class FileTest extends Test {
break;

case TokenKind.TAP_PLAN:
this.reporter.plan(indent, node.end - node.start + 1);
this.reporter.plan(nesting, node.end - node.start + 1);
break;

case TokenKind.TAP_SUBTEST_POINT:
this.reporter.subtest(indent, node.name);
this.reporter.subtest(nesting, node.name);
break;

case TokenKind.TAP_TEST_POINT:
Expand All @@ -172,15 +177,15 @@ class FileTest extends Test {

if (pass) {
this.reporter.ok(
indent,
nesting,
node.id,
node.description,
details(node.diagnostics),
directive
);
} else {
this.reporter.fail(
indent,
nesting,
node.id,
node.description,
details(node.diagnostics),
Expand All @@ -190,15 +195,15 @@ class FileTest extends Test {
break;

case TokenKind.COMMENT:
if (indent === kDefaultIndent) {
if (nesting === 1) {
// Ignore file top level diagnostics
break;
}
this.reporter.diagnostic(indent, node.comment);
this.reporter.diagnostic(nesting, node.comment);
break;

case TokenKind.UNKNOWN:
this.reporter.diagnostic(indent, node.value);
this.reporter.diagnostic(nesting, node.value);
break;
}
}
Expand Down
84 changes: 54 additions & 30 deletions lib/internal/test_runner/tap_stream.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ const {
StringPrototypeReplaceAll,
StringPrototypeToUpperCase,
StringPrototypeSplit,
StringPrototypeRepeat,
RegExpPrototypeSymbolReplace,
} = primordials;
const { inspectWithNoCustomRetry } = require('internal/errors');
const Readable = require('internal/streams/readable');
const { isError, kEmptyObject } = require('internal/util');
const kDefaultIndent = ' '; // 4 spaces
const kFrameStartRegExp = /^ {4}at /;
const kLineBreakRegExp = /\n|\r\n/;
const kDefaultTAPVersion = 13;
Expand All @@ -30,8 +32,8 @@ class TapStream extends Readable {
#buffer;
#canPush;

constructor() {
super();
constructor(options = kEmptyObject) {
super(options);
this.#buffer = [];
this.#canPush = true;
}
Expand All @@ -40,34 +42,30 @@ class TapStream extends Readable {
this.#canPush = true;

while (this.#buffer.length > 0) {
const line = ArrayPrototypeShift(this.#buffer);
const chunk = ArrayPrototypeShift(this.#buffer);

if (!this.#tryPush(line)) {
if (!this.#tryPush(chunk)) {
return;
}
}
}

bail(message) {
this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`);
fail(nesting, testNumber, name, details, directive) {
this.#emit('test:fail', { __proto__: null, name, nesting, testNumber, details, ...directive });
this.#test(nesting, testNumber, 'not ok', name, directive);
this.#details(nesting, details);
}

fail(indent, testNumber, name, details, directive) {
this.emit('test:fail', { __proto__: null, name, testNumber, details, ...directive });
this.#test(indent, testNumber, 'not ok', name, directive);
this.#details(indent, details);
ok(nesting, testNumber, name, details, directive) {
this.#emit('test:pass', { __proto__: null, name, nesting, testNumber, details, ...directive });
this.#test(nesting, testNumber, 'ok', name, directive);
this.#details(nesting, details);
}

ok(indent, testNumber, name, details, directive) {
this.emit('test:pass', { __proto__: null, name, testNumber, details, ...directive });
this.#test(indent, testNumber, 'ok', name, directive);
this.#details(indent, details);
}

plan(indent, count, explanation) {
plan(nesting, count, explanation) {
const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`;

this.#tryPush(`${indent}1..${count}${exp}\n`);
this.#tryPushString(`${this.#indent(nesting)}1..${count}${exp}\n`);
}

getSkip(reason) {
Expand All @@ -78,32 +76,42 @@ class TapStream extends Readable {
return { __proto__: null, todo: reason };
}

subtest(indent, name) {
this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`);
subtest(nesting, name) {
this.#emit('test:subtest', { nesting, name });
this.#tryPushString(`${this.#indent(nesting)}# Subtest: ${tapEscape(name)}\n`);
}

#details(indent, data = kEmptyObject) {
#details(nesting, data = kEmptyObject) {
const { error, duration, yaml } = data;
const indent = this.#indent(nesting);
let details = `${indent} ---\n`;

details += `${yaml ? yaml : ''}`;
details += jsToYaml(indent, 'duration_ms', duration);
details += jsToYaml(indent, null, error);
details += `${indent} ...\n`;
this.#tryPush(details);
this.#tryPushString(details);
}

diagnostic(indent, message) {
this.emit('test:diagnostic', message);
this.#tryPush(`${indent}# ${tapEscape(message)}\n`);
diagnostic(nesting, message) {
this.#emit('test:diagnostic', message);
this.#tryPushString(`${this.#indent(nesting)}# ${tapEscape(message)}\n`);
}

version(spec = kDefaultTAPVersion) {
this.#tryPush(`TAP version ${spec}\n`);
this.#tryPushString(`TAP version ${spec}\n`);
}

#indent(nesting) {
return StringPrototypeRepeat(kDefaultIndent, nesting);
}

#test(indent, testNumber, status, name, directive = kEmptyObject) {
let line = `${indent}${status} ${testNumber}`;
#test(nesting, testNumber, status, name, directive = kEmptyObject) {
if (this._readableState.objectMode) {
// early return
return;
}
let line = `${this.#indent(nesting)}${status} ${testNumber}`;

if (name) {
line += ` ${tapEscape(`- ${name}`)}`;
Expand All @@ -115,7 +123,23 @@ class TapStream extends Readable {

line += '\n';

this.#tryPush(line);
this.#tryPushString(line);
}

#emit(type, data) {
this.emit(type, data);
this.#tryPushObject({ type, data });
}

#tryPushString(str) {
if (!this._readableState.objectMode) {
this.#tryPush(str);
}
}
#tryPushObject(obj) {
if (this._readableState.objectMode) {
this.#tryPush(obj);
}
}

#tryPush(message) {
Expand Down Expand Up @@ -261,4 +285,4 @@ function isAssertionLike(value) {
return value && typeof value === 'object' && 'expected' in value && 'actual' in value;
}

module.exports = { TapStream };
module.exports = { TapStream, kDefaultIndent };
41 changes: 20 additions & 21 deletions lib/internal/test_runner/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,9 @@ const kTestCodeFailure = 'testCodeFailure';
const kTestTimeoutFailure = 'testTimeoutFailure';
const kHookFailure = 'hookFailed';
const kDefaultTimeout = null;
const kDefaultIndent = ' '; // 4 spaces
const noop = FunctionPrototype;
const isTestRunner = getOptionValue('--test');
const hasReporters = Boolean(getOptionValue('--test-reporter'));
const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
const testNamePatternFlag = isTestRunner ? null :
getOptionValue('--test-name-pattern');
Expand Down Expand Up @@ -171,18 +171,18 @@ class Test extends AsyncResource {

if (parent === null) {
this.concurrency = 1;
this.indent = '';
this.nesting = 0;
this.only = testOnlyFlag;
this.reporter = new TapStream();
this.reporter = new TapStream({ objectMode: hasReporters });
this.runOnlySubtests = this.only;
this.testNumber = 0;
this.timeout = kDefaultTimeout;
} else {
const indent = parent.parent === null ? parent.indent :
parent.indent + kDefaultIndent;
const nesting = parent.parent === null ? parent.nesting :
parent.nesting + 1;

this.concurrency = parent.concurrency;
this.indent = indent;
this.nesting = nesting;
this.only = only ?? !parent.runOnlySubtests;
this.reporter = parent.reporter;
this.runOnlySubtests = !this.only;
Expand Down Expand Up @@ -607,19 +607,19 @@ class Test extends AsyncResource {
this.parent.processPendingSubtests();
} else if (!this.reported) {
this.reported = true;
this.reporter.plan(this.indent, this.subtests.length);
this.reporter.plan(this.nesting, this.subtests.length);

for (let i = 0; i < this.diagnostics.length; i++) {
this.reporter.diagnostic(this.indent, this.diagnostics[i]);
this.reporter.diagnostic(this.nesting, this.diagnostics[i]);
}

this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`);
this.reporter.diagnostic(this.indent, `pass ${counters.passed}`);
this.reporter.diagnostic(this.indent, `fail ${counters.failed}`);
this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`);
this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`);
this.reporter.diagnostic(this.indent, `todo ${counters.todo}`);
this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`);
this.reporter.diagnostic(this.nesting, `tests ${this.subtests.length}`);
this.reporter.diagnostic(this.nesting, `pass ${counters.passed}`);
this.reporter.diagnostic(this.nesting, `fail ${counters.failed}`);
this.reporter.diagnostic(this.nesting, `cancelled ${counters.cancelled}`);
this.reporter.diagnostic(this.nesting, `skipped ${counters.skipped}`);
this.reporter.diagnostic(this.nesting, `todo ${counters.todo}`);
this.reporter.diagnostic(this.nesting, `duration_ms ${this.#duration()}`);
this.reporter.push(null);
}
}
Expand Down Expand Up @@ -655,7 +655,7 @@ class Test extends AsyncResource {

report() {
if (this.subtests.length > 0) {
this.reporter.plan(this.subtests[0].indent, this.subtests.length);
this.reporter.plan(this.subtests[0].nesting, this.subtests.length);
} else {
this.reportSubtest();
}
Expand All @@ -669,14 +669,14 @@ class Test extends AsyncResource {
}

if (this.passed) {
this.reporter.ok(this.indent, this.testNumber, this.name, details, directive);
this.reporter.ok(this.nesting, this.testNumber, this.name, details, directive);
} else {
details.error = this.error;
this.reporter.fail(this.indent, this.testNumber, this.name, details, directive);
this.reporter.fail(this.nesting, this.testNumber, this.name, details, directive);
}

for (let i = 0; i < this.diagnostics.length; i++) {
this.reporter.diagnostic(this.indent, this.diagnostics[i]);
this.reporter.diagnostic(this.nesting, this.diagnostics[i]);
}
}

Expand All @@ -686,7 +686,7 @@ class Test extends AsyncResource {
}
this.#reportedSubtest = true;
this.parent.reportSubtest();
this.reporter.subtest(this.indent, this.name);
this.reporter.subtest(this.nesting, this.name);
}
}

Expand Down Expand Up @@ -790,7 +790,6 @@ class Suite extends Test {
module.exports = {
ItTest,
kCancelledByParent,
kDefaultIndent,
kSubtestsFailed,
kTestCodeFailure,
kUnwrapErrors,
Expand Down

0 comments on commit 98598f0

Please sign in to comment.