diff --git a/README.md b/README.md index 66bc7f5..aa93361 100644 --- a/README.md +++ b/README.md @@ -324,6 +324,35 @@ Otherwise, the test is considered to be a failure. Test files must be executable by Node.js, but are not required to use the `node:test` module internally. +## `run([options])` + + + +* `options` {Object} Configuration options for running tests. The following + properties are supported: + * `concurrency` {number|boolean} If a number is provided, + then that many files would run in parallel. + If truthy, it would run (number of cpu cores - 1) + files in parallel. + If falsy, it would only run one file at a time. + If unspecified, subtests inherit this value from their parent. + **Default:** `true`. + * `files`: {Array} An array containing the list of files to run. + **Default** matching files from [test runner execution model][]. + * `signal` {AbortSignal} Allows aborting an in-progress test execution. + * `timeout` {number} A number of milliseconds the test execution will + fail after. + If unspecified, subtests inherit this value from their parent. + **Default:** `Infinity`. +* Returns: {TapStream} + +```js +run({ files: [path.resolve('./tests/test.js')] }) + .pipe(process.stdout); +``` + ## `test([name][, options][, fn])` - `name` {string} The name of the test, which is displayed when reporting test @@ -541,6 +570,47 @@ describe('tests', async () => { }); ``` +## Class: `TapStream` + + + +* Extends {ReadableStream} + +A successful call to [`run()`][] method will return a new {TapStream} +object, streaming a [TAP][] output +`TapStream` will emit events, in the order of the tests definition + +### Event: `'test:diagnostic'` + +* `message` {string} The diagnostic message. + +Emitted when [`context.diagnostic`][] is called. + +### Event: `'test:fail'` + +* `data` {Object} + * `duration` {number} The test duration. + * `error` {Error} The failure casing test to fail. + * `name` {string} The test name. + * `testNumber` {number} The ordinal number of the test. + * `todo` {string|undefined} Present if [`context.todo`][] is called + * `skip` {string|undefined} Present if [`context.skip`][] is called + +Emitted when a test fails. + +### Event: `'test:pass'` + +* `data` {Object} + * `duration` {number} The test duration. + * `name` {string} The test name. + * `testNumber` {number} The ordinal number of the test. + * `todo` {string|undefined} Present if [`context.todo`][] is called + * `skip` {string|undefined} Present if [`context.skip`][] is called + +Emitted when a test passes. + ## Class: `TestContext` An instance of `TestContext` is passed to each test function in order to @@ -712,6 +782,10 @@ The name of the suite. [TAP]: https://testanything.org/ [`SuiteContext`]: #class-suitecontext [`TestContext`]: #class-testcontext +[`context.diagnostic`]: #contextdiagnosticmessage +[`context.skip`]: #contextskipmessage +[`context.todo`]: #contexttodomessage +[`run()`]: #runoptions [`test()`]: #testname-options-fn [describe options]: #describename-options-fn [it options]: #testname-options-fn diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index db41695..ce20a1e 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -1,148 +1,15 @@ -// https://github.com/nodejs/node/blob/2fd4c013c221653da2a7921d08fe1aa96aaba504/lib/internal/main/test_runner.js +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/main/test_runner.js 'use strict' -const { - ArrayFrom, - ArrayPrototypeFilter, - ArrayPrototypeIncludes, - ArrayPrototypeJoin, - ArrayPrototypePush, - ArrayPrototypeSlice, - ArrayPrototypeSort, - SafePromiseAll, - SafeSet -} = require('#internal/per_context/primordials') const { prepareMainThreadExecution -} = require('#internal/bootstrap/pre_execution') -const { spawn } = require('child_process') -const { readdirSync, statSync } = require('fs') -const { - codes: { - ERR_TEST_FAILURE - } -} = require('#internal/errors') -const { toArray } = require('#internal/streams/operators').promiseReturningOperators -const { test } = require('#internal/test_runner/harness') -const { kSubtestsFailed } = require('#internal/test_runner/test') -const { - isSupportedFileType, - doesPathMatchFilter -} = require('#internal/test_runner/utils') -const { basename, join, resolve } = require('path') -const { once } = require('events') -const kFilterArgs = ['--test'] +} = require('#internal/process/pre_execution') +const { run } = require('#internal/test_runner/runner') prepareMainThreadExecution(false) // markBootstrapComplete(); -// TODO(cjihrig): Replace this with recursive readdir once it lands. -function processPath (path, testFiles, options) { - const stats = statSync(path) - - if (stats.isFile()) { - if (options.userSupplied || - (options.underTestDir && isSupportedFileType(path)) || - doesPathMatchFilter(path)) { - testFiles.add(path) - } - } else if (stats.isDirectory()) { - const name = basename(path) - - if (!options.userSupplied && name === 'node_modules') { - return - } - - // 'test' directories get special treatment. Recursively add all .js, - // .cjs, and .mjs files in the 'test' directory. - const isTestDir = name === 'test' - const { underTestDir } = options - const entries = readdirSync(path) - - if (isTestDir) { - options.underTestDir = true - } - - options.userSupplied = false - - for (let i = 0; i < entries.length; i++) { - processPath(join(path, entries[i]), testFiles, options) - } - - options.underTestDir = underTestDir - } -} - -function createTestFileList () { - const cwd = process.cwd() - const hasUserSuppliedPaths = process.argv.length > 1 - const testPaths = hasUserSuppliedPaths - ? ArrayPrototypeSlice(process.argv, 1) - : [cwd] - const testFiles = new SafeSet() - - try { - for (let i = 0; i < testPaths.length; i++) { - const absolutePath = resolve(testPaths[i]) - - processPath(absolutePath, testFiles, { userSupplied: true }) - } - } catch (err) { - if (err?.code === 'ENOENT') { - console.error(`Could not find '${err.path}'`) - process.exit(1) - } - - throw err - } - - return ArrayPrototypeSort(ArrayFrom(testFiles)) -} - -function filterExecArgv (arg) { - return !ArrayPrototypeIncludes(kFilterArgs, arg) -} - -function runTestFile (path) { - return test(path, async (t) => { - const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv) - ArrayPrototypePush(args, path) - - const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' }) - // TODO(cjihrig): Implement a TAP parser to read the child's stdout - // instead of just displaying it all if the child fails. - let err - - child.on('error', (error) => { - err = error - }) - - const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([ - once(child, 'exit', { signal: t.signal }), - toArray.call(child.stdout, { signal: t.signal }), - toArray.call(child.stderr, { signal: t.signal }) - ]) - - if (code !== 0 || signal !== null) { - if (!err) { - err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed) - err.exitCode = code - err.signal = signal - err.stdout = ArrayPrototypeJoin(stdout, '') - err.stderr = ArrayPrototypeJoin(stderr, '') - // The stack will not be useful since the failures came from tests - // in a child process. - err.stack = undefined - } - - throw err - } - }) -} - -;(async function main () { - const testFiles = createTestFileList() - - for (let i = 0; i < testFiles.length; i++) { - runTestFile(testFiles[i]) - } -})() +const tapStream = run() +tapStream.pipe(process.stdout) +tapStream.once('test:fail', () => { + process.exitCode = 1 +}) diff --git a/lib/internal/per_context/primordials.js b/lib/internal/per_context/primordials.js index e599da9..70f1abf 100644 --- a/lib/internal/per_context/primordials.js +++ b/lib/internal/per_context/primordials.js @@ -3,6 +3,8 @@ const replaceAll = require('string.prototype.replaceall') exports.ArrayFrom = (it, mapFn) => Array.from(it, mapFn) +exports.ArrayIsArray = Array.isArray +exports.ArrayPrototypeConcat = (arr, ...el) => arr.concat(...el) exports.ArrayPrototypeFilter = (arr, fn) => arr.filter(fn) exports.ArrayPrototypeForEach = (arr, fn, thisArg) => arr.forEach(fn, thisArg) exports.ArrayPrototypeIncludes = (arr, el, fromIndex) => arr.includes(el, fromIndex) @@ -20,6 +22,7 @@ exports.FunctionPrototype = Function.prototype exports.FunctionPrototypeBind = (fn, obj, ...args) => fn.bind(obj, ...args) exports.MathMax = (...args) => Math.max(...args) exports.Number = Number +exports.ObjectAssign = (target, ...sources) => Object.assign(target, ...sources) exports.ObjectCreate = obj => Object.create(obj) exports.ObjectDefineProperties = (obj, props) => Object.defineProperties(obj, props) exports.ObjectDefineProperty = (obj, key, descr) => Object.defineProperty(obj, key, descr) @@ -41,6 +44,7 @@ exports.SafePromiseAll = (array, mapFn) => Promise.all(mapFn ? array.map(mapFn) exports.SafePromiseRace = (array, mapFn) => Promise.race(mapFn ? array.map(mapFn) : array) exports.SafeSet = Set exports.SafeWeakMap = WeakMap +exports.SafeWeakSet = WeakSet exports.StringPrototypeIncludes = (str, needle) => str.includes(needle) exports.StringPrototypeMatch = (str, reg) => str.match(reg) exports.StringPrototypeReplace = (str, search, replacement) => diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/process/pre_execution.js similarity index 100% rename from lib/internal/bootstrap/pre_execution.js rename to lib/internal/process/pre_execution.js diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 987de95..bddfedc 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -1,9 +1,9 @@ -// https://github.com/nodejs/node/blob/8cf33850bea691d8c53b2d4175c959c8549aa76c/lib/internal/test_runner/harness.js +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/test_runner/harness.js 'use strict' const { ArrayPrototypeForEach, - FunctionPrototypeBind, - SafeMap + SafeMap, + SafeWeakSet } = require('#internal/per_context/primordials') const { createHook, @@ -14,13 +14,18 @@ const { ERR_TEST_FAILURE } } = require('#internal/errors') +const { kEmptyObject } = require('#internal/util') const { getOptionValue } = require('#internal/options') const { kCancelledByParent, Test, ItTest, Suite } = require('#internal/test_runner/test') +const { bigint: hrtime } = process.hrtime -const isTestRunner = getOptionValue('--test') +const isTestRunnerCli = getOptionValue('--test') const testResources = new SafeMap() -const root = new Test({ __proto__: null, name: '' }) -let wasRootSetup = false +const wasRootSetup = new SafeWeakSet() + +function createTestTree (options = kEmptyObject) { + return setup(new Test({ __proto__: null, ...options, name: '' })) +} function createProcessEventHandler (eventName, rootTest) { return (err) => { @@ -51,7 +56,7 @@ function createProcessEventHandler (eventName, rootTest) { } function setup (root) { - if (wasRootSetup) { + if (wasRootSetup.has(root)) { return root } const hook = createHook({ @@ -84,52 +89,9 @@ function setup (root) { 'Promise resolution is still pending but the event loop has already resolved', kCancelledByParent)) - let passCount = 0 - let failCount = 0 - let skipCount = 0 - let todoCount = 0 - let cancelledCount = 0 - - for (let i = 0; i < root.subtests.length; i++) { - const test = root.subtests[i] - - // Check SKIP and TODO tests first, as those should not be counted as - // failures. - if (test.skipped) { - skipCount++ - } else if (test.isTodo) { - todoCount++ - } else if (test.cancelled) { - cancelledCount++ - } else if (!test.passed) { - failCount++ - } else { - passCount++ - } - } - - root.reporter.plan(root.indent, root.subtests.length) - - for (let i = 0; i < root.diagnostics.length; i++) { - root.reporter.diagnostic(root.indent, root.diagnostics[i]) - } - - root.reporter.diagnostic(root.indent, `tests ${root.subtests.length}`) - root.reporter.diagnostic(root.indent, `pass ${passCount}`) - root.reporter.diagnostic(root.indent, `fail ${failCount}`) - root.reporter.diagnostic(root.indent, `cancelled ${cancelledCount}`) - root.reporter.diagnostic(root.indent, `skipped ${skipCount}`) - root.reporter.diagnostic(root.indent, `todo ${todoCount}`) - root.reporter.diagnostic(root.indent, `duration_ms ${process.uptime()}`) - - root.reporter.push(null) hook.disable() process.removeListener('unhandledRejection', rejectionHandler) process.removeListener('uncaughtException', exceptionHandler) - - if (failCount > 0 || cancelledCount > 0) { - process.exitCode = 1 - } } const terminationHandler = () => { @@ -140,29 +102,41 @@ function setup (root) { process.on('uncaughtException', exceptionHandler) process.on('unhandledRejection', rejectionHandler) process.on('beforeExit', exitHandler) - // TODO(MoLow): Make it configurable to hook when isTestRunner === false. - if (isTestRunner) { + // TODO(MoLow): Make it configurable to hook when isTestRunnerCli === false. + if (isTestRunnerCli) { process.on('SIGINT', terminationHandler) process.on('SIGTERM', terminationHandler) } - root.reporter.pipe(process.stdout) + root.startTime = hrtime() root.reporter.version() - wasRootSetup = true + wasRootSetup.add(root) return root } +let globalRoot +function getGlobalRoot () { + if (!globalRoot) { + globalRoot = createTestTree() + globalRoot.reporter.pipe(process.stdout) + globalRoot.reporter.once('test:fail', () => { + process.exitCode = 1 + }) + } + return globalRoot +} + function test (name, options, fn) { - const subtest = setup(root).createSubtest(Test, name, options, fn) + const subtest = getGlobalRoot().createSubtest(Test, name, options, fn) return subtest.start() } function runInParentContext (Factory) { function run (name, options, fn, overrides) { - const parent = testResources.get(executionAsyncId()) || setup(root) + const parent = testResources.get(executionAsyncId()) || getGlobalRoot() const subtest = parent.createSubtest(Factory, name, options, fn, overrides) - if (parent === root) { + if (parent === getGlobalRoot()) { subtest.start() } } @@ -181,13 +155,14 @@ function runInParentContext (Factory) { function hook (hook) { return (fn, options) => { - const parent = testResources.get(executionAsyncId()) || setup(root) + const parent = testResources.get(executionAsyncId()) || getGlobalRoot() parent.createHook(hook, fn, options) } } module.exports = { - test: FunctionPrototypeBind(test, root), + createTestTree, + test, describe: runInParentContext(Suite), it: runInParentContext(ItTest), before: hook('before'), diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js new file mode 100644 index 0000000..8266087 --- /dev/null +++ b/lib/internal/test_runner/runner.js @@ -0,0 +1,165 @@ +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/test_runner/runner.js +'use strict' +const { + ArrayFrom, + ArrayPrototypeConcat, + ArrayPrototypeFilter, + ArrayPrototypeIncludes, + ArrayPrototypeJoin, + ArrayPrototypeSlice, + ArrayPrototypeSort, + ObjectAssign, + PromisePrototypeThen, + SafePromiseAll, + SafeSet +} = require('#internal/per_context/primordials') + +const { spawn } = require('child_process') +const { readdirSync, statSync } = require('fs') +const { + codes: { + ERR_TEST_FAILURE + } +} = require('#internal/errors') +const { toArray } = require('#internal/streams/operators').promiseReturningOperators +const { validateArray } = require('#internal/validators') +const { kEmptyObject } = require('#internal/util') +const { createTestTree } = require('#internal/test_runner/harness') +const { kSubtestsFailed, Test } = require('#internal/test_runner/test') +const { + isSupportedFileType, + doesPathMatchFilter +} = require('#internal/test_runner/utils') +const { basename, join, resolve } = require('path') +const { once } = require('events') + +const kFilterArgs = ['--test'] + +// TODO(cjihrig): Replace this with recursive readdir once it lands. +function processPath (path, testFiles, options) { + const stats = statSync(path) + + if (stats.isFile()) { + if (options.userSupplied || + (options.underTestDir && isSupportedFileType(path)) || + doesPathMatchFilter(path)) { + testFiles.add(path) + } + } else if (stats.isDirectory()) { + const name = basename(path) + + if (!options.userSupplied && name === 'node_modules') { + return + } + + // 'test' directories get special treatment. Recursively add all .js, + // .cjs, and .mjs files in the 'test' directory. + const isTestDir = name === 'test' + const { underTestDir } = options + const entries = readdirSync(path) + + if (isTestDir) { + options.underTestDir = true + } + + options.userSupplied = false + + for (let i = 0; i < entries.length; i++) { + processPath(join(path, entries[i]), testFiles, options) + } + + options.underTestDir = underTestDir + } +} + +function createTestFileList () { + const cwd = process.cwd() + const hasUserSuppliedPaths = process.argv.length > 1 + const testPaths = hasUserSuppliedPaths + ? ArrayPrototypeSlice(process.argv, 1) + : [cwd] + const testFiles = new SafeSet() + + try { + for (let i = 0; i < testPaths.length; i++) { + const absolutePath = resolve(testPaths[i]) + + processPath(absolutePath, testFiles, { userSupplied: true }) + } + } catch (err) { + if (err?.code === 'ENOENT') { + console.error(`Could not find '${err.path}'`) + process.exit(1) + } + + throw err + } + + return ArrayPrototypeSort(ArrayFrom(testFiles)) +} + +function filterExecArgv (arg) { + return !ArrayPrototypeIncludes(kFilterArgs, arg) +} + +function runTestFile (path, root) { + const subtest = root.createSubtest(Test, path, async (t) => { + const args = ArrayPrototypeConcat( + ArrayPrototypeFilter(process.execArgv, filterExecArgv), + path) + + const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' }) + // TODO(cjihrig): Implement a TAP parser to read the child's stdout + // instead of just displaying it all if the child fails. + let err + + child.on('error', (error) => { + err = error + }) + + const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([ + once(child, 'exit', { signal: t.signal }), + toArray.call(child.stdout, { signal: t.signal }), + toArray.call(child.stderr, { signal: t.signal }) + ]) + + if (code !== 0 || signal !== null) { + if (!err) { + err = ObjectAssign(new ERR_TEST_FAILURE('test failed', kSubtestsFailed), { + __proto__: null, + exitCode: code, + signal, + stdout: ArrayPrototypeJoin(stdout, ''), + stderr: ArrayPrototypeJoin(stderr, ''), + // The stack will not be useful since the failures came from tests + // in a child process. + stack: undefined + }) + } + + throw err + } + }) + return subtest.start() +} + +function run (options) { + if (options === null || typeof options !== 'object') { + options = kEmptyObject + } + const { concurrency, timeout, signal, files } = options + + if (files != null) { + validateArray(files, 'options.files') + } + + const root = createTestTree({ concurrency, timeout, signal }) + const testFiles = files ?? createTestFileList() + + PromisePrototypeThen(SafePromiseAll(testFiles, (path) => runTestFile(path, root)), + () => root.postRun()) + + return root.reporter +} + +module.exports = { run } diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index e240598..769d275 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -1,10 +1,11 @@ -// https://github.com/nodejs/node/blob/0d46cf6af8977d1e2e4c4886bf0f7e0dbe76d21c/lib/internal/test_runner/tap_stream.js +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/test_runner/tap_stream.js 'use strict' const { ArrayPrototypeForEach, ArrayPrototypeJoin, + ArrayPrototypeMap, ArrayPrototypePush, ArrayPrototypeShift, ObjectEntries, @@ -14,7 +15,7 @@ const { } = require('#internal/per_context/primordials') const { inspectWithNoCustomRetry } = require('#internal/errors') const Readable = require('#internal/streams/readable') -const { isError } = require('#internal/util') +const { isError, kEmptyObject } = require('#internal/util') const kFrameStartRegExp = /^ {4}at / const kLineBreakRegExp = /\n|\r\n/ const inspectOptions = { colors: false, breakLength: Infinity } @@ -53,12 +54,16 @@ class TapStream extends Readable { this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`) } - fail (indent, testNumber, description, directive) { - this.#test(indent, testNumber, 'not ok', description, directive) + fail (indent, testNumber, name, duration, error, directive) { + this.emit('test:fail', { __proto__: null, name, testNumber, duration, ...directive, error }) + this.#test(indent, testNumber, 'not ok', name, directive) + this.#details(indent, duration, error) } - ok (indent, testNumber, description, directive) { - this.#test(indent, testNumber, 'ok', description, directive) + ok (indent, testNumber, name, duration, directive) { + this.emit('test:pass', { __proto__: null, name, testNumber, duration, ...directive }) + this.#test(indent, testNumber, 'ok', name, directive) + this.#details(indent, duration, null) } plan (indent, count, explanation) { @@ -68,18 +73,18 @@ class TapStream extends Readable { } getSkip (reason) { - return `SKIP${reason ? ` ${tapEscape(reason)}` : ''}` + return { __proto__: null, skip: reason } } getTodo (reason) { - return `TODO${reason ? ` ${tapEscape(reason)}` : ''}` + return { __proto__: null, todo: reason } } subtest (indent, name) { this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`) } - details (indent, duration, error) { + #details (indent, duration, error) { let details = `${indent} ---\n` details += jsToYaml(indent, 'duration_ms', duration) @@ -89,6 +94,7 @@ class TapStream extends Readable { } diagnostic (indent, message) { + this.emit('test:diagnostic', message) this.#tryPush(`${indent}# ${tapEscape(message)}\n`) } @@ -96,16 +102,16 @@ class TapStream extends Readable { this.#tryPush('TAP version 13\n') } - #test (indent, testNumber, status, description, directive) { + #test (indent, testNumber, status, name, directive = kEmptyObject) { let line = `${indent}${status} ${testNumber}` - if (description) { - line += ` ${tapEscape(description)}` + if (name) { + line += ` ${tapEscape(`- ${name}`)}` } - if (directive) { - line += ` # ${directive}` - } + line += ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(directive), ({ 0: key, 1: value }) => ( + ` # ${key.toUpperCase()}${value ? ` ${tapEscape(value)}` : ''}` + )), '') line += '\n' this.#tryPush(line) diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index b4fac63..20c1d98 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1,4 +1,4 @@ -// https://github.com/nodejs/node/blob/0d46cf6af8977d1e2e4c4886bf0f7e0dbe76d21c/lib/internal/test_runner/test.js +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/test_runner/test.js 'use strict' @@ -179,7 +179,7 @@ class Test extends AsyncResource { case 'boolean': if (concurrency) { - this.concurrency = isTestRunner ? MathMax(cpus().length - 1, 1) : Infinity + this.concurrency = parent === null ? MathMax(cpus().length - 1, 1) : Infinity } else { this.concurrency = 1 } @@ -514,7 +514,7 @@ class Test extends AsyncResource { } postRun (pendingSubtestsError) { - let failedSubtests = 0 + const counters = { __proto__: null, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 } // If the test was failed before it even started, then the end time will // be earlier than the start time. Correct that here. @@ -534,14 +534,28 @@ class Test extends AsyncResource { subtest.postRun(pendingSubtestsError) } + // Check SKIP and TODO tests first, as those should not be counted as + // failures. + if (subtest.skipped) { + counters.skipped++ + } else if (subtest.isTodo) { + counters.todo++ + } else if (subtest.cancelled) { + counters.cancelled++ + } else if (!subtest.passed) { + counters.failed++ + } else { + counters.passed++ + } + if (!subtest.passed) { - failedSubtests++ + counters.totalFailed++ } } - if (this.passed && failedSubtests > 0) { - const subtestString = `subtest${failedSubtests > 1 ? 's' : ''}` - const msg = `${failedSubtests} ${subtestString} failed` + if ((this.passed || this.parent === null) && counters.totalFailed > 0) { + const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}` + const msg = `${counters.totalFailed} ${subtestString} failed` this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed)) } @@ -553,6 +567,22 @@ class Test extends AsyncResource { this.parent.addReadySubtest(this) this.parent.processReadySubtestRange(false) this.parent.processPendingSubtests() + } else if (!this.reported) { + this.reported = true + this.reporter.plan(this.indent, this.subtests.length) + + for (let i = 0; i < this.diagnostics.length; i++) { + this.reporter.diagnostic(this.indent, this.diagnostics[i]) + } + + this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`) + this.reporter.diagnostic(this.indent, `pass ${counters.passed}`) + this.reporter.diagnostic(this.indent, `fail ${counters.failed}`) + this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`) + this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`) + this.reporter.diagnostic(this.indent, `todo ${counters.todo}`) + this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`) + this.reporter.push(null) } } @@ -586,10 +616,12 @@ class Test extends AsyncResource { this.finished = true } - report () { + #duration () { // Duration is recorded in BigInt nanoseconds. Convert to seconds. - const duration = Number(this.endTime - this.startTime) / 1_000_000_000 - const message = `- ${this.name}` + return Number(this.endTime - this.startTime) / 1_000_000_000 + } + + report () { let directive if (this.skipped) { @@ -599,13 +631,11 @@ class Test extends AsyncResource { } if (this.passed) { - this.reporter.ok(this.indent, this.testNumber, message, directive) + this.reporter.ok(this.indent, this.testNumber, this.name, this.#duration(), directive) } else { - this.reporter.fail(this.indent, this.testNumber, message, directive) + this.reporter.fail(this.indent, this.testNumber, this.name, this.#duration(), this.error, directive) } - this.reporter.details(this.indent, duration, this.error) - for (let i = 0; i < this.diagnostics.length; i++) { this.reporter.diagnostic(this.indent, this.diagnostics[i]) } @@ -630,6 +660,9 @@ class TestHook extends Test { getRunArgs () { return this.#args } + + postRun () { + } } class ItTest extends Test { diff --git a/lib/internal/validators.js b/lib/internal/validators.js index 511e61b..06e5746 100644 --- a/lib/internal/validators.js +++ b/lib/internal/validators.js @@ -1,5 +1,6 @@ // https://github.com/nodejs/node/blob/60da0a1b364efdd84870269d23b39faa12fb46d8/lib/internal/validators.js const { + ArrayIsArray, ArrayPrototypeIncludes, ArrayPrototypeJoin, ArrayPrototypeMap @@ -64,9 +65,20 @@ const validateOneOf = (value, name, oneOf) => { } } +const validateArray = (value, name, minLength = 0) => { + if (!ArrayIsArray(value)) { + throw new ERR_INVALID_ARG_TYPE(name, 'Array', value) + } + if (value.length < minLength) { + const reason = `must be longer than ${minLength}` + throw new ERR_INVALID_ARG_VALUE(name, value, reason) + } +} + module.exports = { isUint32, validateAbortSignal, + validateArray, validateNumber, validateOneOf, validateUint32 diff --git a/lib/test.js b/lib/test.js index a65fb3c..1662780 100644 --- a/lib/test.js +++ b/lib/test.js @@ -1,12 +1,16 @@ -// https://github.com/nodejs/node/blob/659dc126932f986fc33c7f1c878cb2b57a1e2fac/lib/test.js 'use strict' +const { ObjectAssign } = require('#internal/per_context/primordials') const { test, describe, it, before, after, beforeEach, afterEach } = require('#internal/test_runner/harness') +const { run } = require('#internal/test_runner/runner') module.exports = test -module.exports.test = test -module.exports.describe = describe -module.exports.it = it -module.exports.before = before -module.exports.after = after -module.exports.beforeEach = beforeEach -module.exports.afterEach = afterEach +ObjectAssign(module.exports, { + after, + afterEach, + before, + beforeEach, + describe, + it, + run, + test +}) diff --git a/test/common/index.js b/test/common/index.js index 91fcd41..a6269ac 100644 --- a/test/common/index.js +++ b/test/common/index.js @@ -37,6 +37,30 @@ function mustCall (fn, exact) { return _mustCallInner(fn, exact, 'exact') } +function getCallSite (top) { + const originalStackFormatter = Error.prepareStackTrace + Error.prepareStackTrace = (err, stack) => // eslint-disable-line n/handle-callback-err + `${stack[0].getFileName()}:${stack[0].getLineNumber()}` + const err = new Error() + Error.captureStackTrace(err, top) + // With the V8 Error API, the stack is not formatted until it is accessed + err.stack // eslint-disable-line no-unused-expressions + Error.prepareStackTrace = originalStackFormatter + return err.stack +} + +function mustNotCall (msg) { + const callSite = getCallSite(mustNotCall) + return function mustNotCall (...args) { + const argsInfo = args.length > 0 + ? `\ncalled with arguments: ${args.map((arg) => util.inspect(arg)).join(', ')}` + : '' + assert.fail( + `${msg || 'function should not have been called'} at ${callSite}` + + argsInfo) + } +} + function _mustCallInner (fn, criteria = 1, field) { if (process._exiting) { throw new Error('Cannot use common.mustCall*() in process exit handler') } if (typeof fn === 'number') { @@ -145,5 +169,6 @@ if (typeof AbortSignal !== 'undefined' && (process.version.startsWith('v14.') || module.exports = { expectsError, isWindow: process.platform === 'win32', - mustCall + mustCall, + mustNotCall } diff --git a/test/message/test_runner_no_tests.out b/test/message/test_runner_no_tests.out index 9f84e58..9daeafb 100644 --- a/test/message/test_runner_no_tests.out +++ b/test/message/test_runner_no_tests.out @@ -1 +1 @@ -bound test +test diff --git a/test/parallel.mjs b/test/parallel.mjs index 48e0f61..e0a5050 100755 --- a/test/parallel.mjs +++ b/test/parallel.mjs @@ -10,7 +10,7 @@ const PARALLEL_DIR = new URL('./parallel/', import.meta.url) const dir = await fs.opendir(PARALLEL_DIR) for await (const { name } of dir) { - if (!name.endsWith('.js')) continue + if (!name.endsWith('.js') && !name.endsWith('.mjs')) continue const cp = spawn( process.execPath, [fileURLToPath(new URL(name, PARALLEL_DIR))], diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs new file mode 100644 index 0000000..66837ae --- /dev/null +++ b/test/parallel/test-runner-run.mjs @@ -0,0 +1,75 @@ +// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/test/parallel/test-runner-run.mjs + +import common from '../common/index.js' +import fixtures from '../common/fixtures.js' +import { join } from 'node:path' +import pkg from '#node:test' +import assert from 'node:assert' + +const { describe, it, run } = pkg + +const testFixtures = fixtures.path('test-runner') + +describe('require(\'node:test\').run', { concurrency: true }, () => { + it('should run with no tests', async () => { + const stream = run({ files: [] }) + stream.setEncoding('utf8') + stream.on('test:fail', common.mustNotCall()) + stream.on('test:pass', common.mustNotCall()) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should fail with non existing file', async () => { + const stream = run({ files: ['a-random-file-that-does-not-exist.js'] }) + stream.on('test:fail', common.mustCall(1)) + stream.on('test:pass', common.mustNotCall()) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should succeed with a file', async () => { + const stream = run({ files: [join(testFixtures, 'test/random.cjs')] }) + stream.on('test:fail', common.mustNotCall()) + stream.on('test:pass', common.mustCall(1)) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should run same file twice', async () => { + const stream = run({ files: [join(testFixtures, 'test/random.cjs'), join(testFixtures, 'test/random.cjs')] }) + stream.on('test:fail', common.mustNotCall()) + stream.on('test:pass', common.mustCall(2)) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should run a failed test', async () => { + const stream = run({ files: [testFixtures] }) + stream.on('test:fail', common.mustCall(1)) + stream.on('test:pass', common.mustNotCall()) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should support timeout', async () => { + const stream = run({ + timeout: 50, + files: [ + fixtures.path('test-runner', 'never_ending_sync.js'), + fixtures.path('test-runner', 'never_ending_async.js') + ] + }) + stream.on('test:fail', common.mustCall(2)) + stream.on('test:pass', common.mustNotCall()) + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); // TODO(MoLow): assert.snapshot + }) + + it('should validate files', async () => { + [Symbol(''), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve([]), true, false] + .forEach((files) => assert.throws(() => run({ files }), { + code: 'ERR_INVALID_ARG_TYPE' + })) + }) +})