Skip to content

Commit

Permalink
test_runner: flatten TAP output when running using --test
Browse files Browse the repository at this point in the history
  • Loading branch information
@MoLow
MoLow committed Jan 31, 2023
1 parent 3abd559 commit 602e6bb
Show file tree
Hide file tree
Showing 7 changed files with 677 additions and 690 deletions.
67 changes: 33 additions & 34 deletions lib/internal/test_runner/runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ const {
ArrayPrototypeSlice,
ArrayPrototypeSome,
ArrayPrototypeSort,
Number,
ObjectAssign,
PromisePrototypeThen,
SafePromiseAll,
SafePromiseAllReturnVoid,
SafePromiseAllSettledReturnVoid,
SafeMap,
Expand All @@ -33,7 +33,7 @@ const { validateArray, validateBoolean } = require('internal/validators');
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
const { kEmptyObject } = require('internal/util');
const { createTestTree } = require('internal/test_runner/harness');
const { kSubtestsFailed, Test } = require('internal/test_runner/test');
const { kSubtestsFailed, kTestCodeFailure, kCancelledByParent, Test } = require('internal/test_runner/test');
const { TapParser } = require('internal/test_runner/tap_parser');
const { YAMLToJs } = require('internal/test_runner/yaml_to_js');
const { TokenKind } = require('internal/test_runner/tap_lexer');
Expand Down Expand Up @@ -130,27 +130,32 @@ function getRunArgs({ path, inspectPort }) {

class FileTest extends Test {
#buffer = [];
#hasSubtests = false;
get skipReporting() {
return this.#hasSubtests && (!this.error || this.error.failureType === kSubtestsFailed);
}
#handleReportItem({ kind, node, nesting = 0 }) {
nesting += 1;

switch (kind) {
case TokenKind.TAP_VERSION:
// TODO(manekinekko): handle TAP version coming from the parser.
// this.reporter.version(node.version);
break;

case TokenKind.TAP_PLAN:
if (nesting === 0 && this.skipReporting) {
break;
}
this.reporter.plan(nesting, this.name, node.end - node.start + 1);
break;

case TokenKind.TAP_SUBTEST_POINT:
this.reporter.start(nesting, this.name, node.name);
break;

case TokenKind.TAP_TEST_POINT:
// eslint-disable-next-line no-case-declarations
case TokenKind.TAP_TEST_POINT: {

const { todo, skip, pass } = node.status;
// eslint-disable-next-line no-case-declarations

let directive;

if (skip) {
Expand All @@ -161,29 +166,20 @@ class FileTest extends Test {
directive = kEmptyObject;
}

if (pass) {
this.reporter.ok(
nesting,
this.name,
node.id,
node.description,
YAMLToJs(node.diagnostics),
directive
);
} else {
this.reporter.fail(
nesting,
this.name,
node.id,
node.description,
YAMLToJs(node.diagnostics),
directive
);
this.#hasSubtests = true;
const diagnostics = YAMLToJs(node.diagnostics);
const cancelled = diagnostics.error?.failureType === kCancelledByParent;
const testNumber = nesting === 0 ? (Number(node.id) + this.testNumber - 1) : node.id;
const method = pass ? 'ok' : 'fail';
this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
if (nesting === 0) {
this.parent.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
this.countSubtest({ finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled });
}
break;

case TokenKind.COMMENT:
if (nesting === 1) {
} case TokenKind.COMMENT:
if (nesting === 0) {
// Ignore file top level diagnostics
break;
}
Expand All @@ -203,10 +199,15 @@ class FileTest extends Test {
this.reportStarted();
this.#handleReportItem(ast);
}
reportStarted() {}
report() {
this.reportStarted();
if (!this.skipReporting) {
super.reportStarted();
}
ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast));
super.report();
if (!this.skipReporting) {
super.report();
}
}
}

Expand Down Expand Up @@ -261,16 +262,14 @@ function runTestFile(path, root, inspectPort, filesWatcher) {
subtest.addToReport(ast);
});

const { 0: { 0: code, 1: signal } } = await SafePromiseAll([
once(child, 'exit', { signal: t.signal }),
child.stdout.toArray({ signal: t.signal }),
]);
const { 0: code, 1: signal } = await once(child, 'exit', { signal: t.signal });

runningProcesses.delete(path);
runningSubtests.delete(path);
if (code !== 0 || signal !== null) {
if (!err) {
err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), {
const failureType = subtest.completedCounters.totalFailed > 0 ? kSubtestsFailed : kTestCodeFailure;
err = ObjectAssign(new ERR_TEST_FAILURE('test failed', failureType), {
__proto__: null,
exitCode: code,
signal: signal,
Expand Down
70 changes: 39 additions & 31 deletions lib/internal/test_runner/test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,9 @@ class Test extends AsyncResource {
beforeEach: [],
afterEach: [],
};
this.completedCounters = {
all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0
};
this.waitingOn = 0;
this.finished = false;
}
Expand Down Expand Up @@ -577,9 +580,32 @@ class Test extends AsyncResource {
this.postRun();
}

postRun(pendingSubtestsError) {
const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
countSubtest(subtest) {
if (subtest.skipReporting || subtest.counted) {
return;
}
subtest.counted = true;
// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (subtest.skipped) {
this.completedCounters.skipped++;
} else if (subtest.isTodo) {
this.completedCounters.todo++;
} else if (subtest.cancelled) {
this.completedCounters.cancelled++;
} else if (!subtest.passed) {
this.completedCounters.failed++;
} else {
this.completedCounters.passed++;
}

if (!subtest.passed) {
this.completedCounters.totalFailed++;
}
this.completedCounters.all++;
}

postRun(pendingSubtestsError) {
// If the test was failed before it even started, then the end time will
// be earlier than the start time. Correct that here.
if (this.endTime < this.startTime) {
Expand All @@ -592,34 +618,15 @@ class Test extends AsyncResource {
this.pendingSubtests = [];
for (let i = 0; i < this.subtests.length; i++) {
const subtest = this.subtests[i];

if (!subtest.finished) {
subtest.cancel(pendingSubtestsError);
subtest.postRun(pendingSubtestsError);
}

// Check SKIP and TODO tests first, as those should not be counted as
// failures.
if (subtest.skipped) {
counters.skipped++;
} else if (subtest.isTodo) {
counters.todo++;
} else if (subtest.cancelled) {
counters.cancelled++;
} else if (!subtest.passed) {
counters.failed++;
} else {
counters.passed++;
}

if (!subtest.passed) {
counters.totalFailed++;
}
}

if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
const msg = `${counters.totalFailed} ${subtestString} failed`;
if ((this.passed || this.parent === null) && this.completedCounters.totalFailed > 0) {
const subtestString = `subtest${this.completedCounters.totalFailed > 1 ? 's' : ''}`;
const msg = `${this.completedCounters.totalFailed} ${subtestString} failed`;

this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
}
Expand All @@ -632,20 +639,21 @@ class Test extends AsyncResource {
this.parent.addReadySubtest(this);
this.parent.processReadySubtestRange(false);
this.parent.processPendingSubtests();
this.parent.countSubtest(this);
} else if (!this.reported) {
this.reported = true;
this.reporter.plan(this.nesting, kFilename, this.subtests.length);
this.reporter.plan(this.nesting, kFilename, this.completedCounters.all);

for (let i = 0; i < this.diagnostics.length; i++) {
this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
}

this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.subtests.length}`);
this.reporter.diagnostic(this.nesting, kFilename, `pass ${counters.passed}`);
this.reporter.diagnostic(this.nesting, kFilename, `fail ${counters.failed}`);
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${counters.cancelled}`);
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${counters.skipped}`);
this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`);
this.reporter.diagnostic(this.nesting, kFilename, `tests ${this.completedCounters.all}`);
this.reporter.diagnostic(this.nesting, kFilename, `pass ${this.completedCounters.passed}`);
this.reporter.diagnostic(this.nesting, kFilename, `fail ${this.completedCounters.failed}`);
this.reporter.diagnostic(this.nesting, kFilename, `cancelled ${this.completedCounters.cancelled}`);
this.reporter.diagnostic(this.nesting, kFilename, `skipped ${this.completedCounters.skipped}`);
this.reporter.diagnostic(this.nesting, kFilename, `todo ${this.completedCounters.todo}`);
this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`);

if (this.coverage) {
Expand Down

0 comments on commit 602e6bb

Please sign in to comment.