From 33a787873f3576beac9b57d817c77c0ab2baedfe Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 6 May 2020 16:09:34 +0800 Subject: [PATCH] test: refactor WPTRunner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Print test results as soon as they are available, instead of until after all the tests are complete. This helps us printing tests whose completion callback is not called because of failures. - Run the scripts specified by `// META: script=` one by one instead of concatenating them first for better error stack traces. - Print a status summary when the test process is about to exit. This can be used as reference for updating the status file. For example the stderr output of `out/Release/node test/wpt/test-console.js` would be: ``` { 'idlharness.any.js': { fail: { expected: [ 'assert_equals: operation has wrong .length expected 1 but got 0' ] } } } Ran 4/4 tests, 0 skipped, 3 passed, 1 expected failures, 0 unexpected failures ``` PR-URL: https://github.com/nodejs/node/pull/33297 Reviewed-By: Anna Henningsen Reviewed-By: Benjamin Gruenbaum Reviewed-By: Michaël Zasso --- test/common/wpt.js | 353 ++++++++++++++++++++++++++------------------- 1 file changed, 208 insertions(+), 145 deletions(-) diff --git a/test/common/wpt.js b/test/common/wpt.js index f6ac07bdfd296c..e33d0c86cc7ed9 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -42,6 +42,18 @@ class ResourceLoader { this.path = path; } + toRealFilePath(from, url) { + // We need to patch this to load the WebIDL parser + url = url.replace( + '/resources/WebIDLParser.js', + '/resources/webidl2/lib/webidl2.js' + ); + const base = path.dirname(from); + return url.startsWith('/') ? + fixtures.path('wpt', url) : + fixtures.path('wpt', base, url); + } + /** * Load a resource in test/fixtures/wpt specified with a URL * @param {string} from the path of the file loading this resource, @@ -51,15 +63,7 @@ class ResourceLoader { * pseudo-Response object. */ read(from, url, asFetch = true) { - // We need to patch this to load the WebIDL parser - url = url.replace( - '/resources/WebIDLParser.js', - '/resources/webidl2/lib/webidl2.js' - ); - const base = path.dirname(from); - const file = url.startsWith('/') ? - fixtures.path('wpt', url) : - fixtures.path('wpt', base, url); + const file = this.toRealFilePath(from, url); if (asFetch) { return fsPromises.readFile(file) .then((data) => { @@ -135,7 +139,8 @@ class StatusRuleSet { } } -class WPTTest { +// A specification of WPT test +class WPTTestSpec { /** * @param {string} mod name of the WPT module, e.g. * 'html/webappapis/microtask-queuing' @@ -227,8 +232,8 @@ class StatusLoader { this.path = path; this.loaded = false; this.rules = new StatusRuleSet(); - /** @type {WPTTest[]} */ - this.tests = []; + /** @type {WPTTestSpec[]} */ + this.specs = []; } /** @@ -265,15 +270,19 @@ class StatusLoader { for (const file of list) { const relativePath = path.relative(subDir, file); const match = this.rules.match(relativePath); - this.tests.push(new WPTTest(this.path, relativePath, match)); + this.specs.push(new WPTTestSpec(this.path, relativePath, match)); } this.loaded = true; } } -const PASSED = 1; -const FAILED = 2; -const SKIPPED = 3; +const kPass = 'pass'; +const kFail = 'fail'; +const kSkip = 'skip'; +const kTimeout = 'timeout'; +const kIncomplete = 'incomplete'; +const kUncaught = 'uncaught'; +const NODE_UNCAUGHT = 100; class WPTRunner { constructor(path) { @@ -286,12 +295,13 @@ class WPTRunner { this.status = new StatusLoader(path); this.status.load(); - this.tests = new Map( - this.status.tests.map((item) => [item.filename, item]) + this.specMap = new Map( + this.status.specs.map((item) => [item.filename, item]) ); - this.results = new Map(); + this.results = {}; this.inProgress = new Set(); + this.unexpectedFailures = []; } /** @@ -328,39 +338,97 @@ class WPTRunner { // only `subset.any.js` will be run by the runner. if (process.argv[2]) { const filename = process.argv[2]; - if (!this.tests.has(filename)) { + if (!this.specMap.has(filename)) { throw new Error(`${filename} not found!`); } - queue.push(this.tests.get(filename)); + queue.push(this.specMap.get(filename)); } else { queue = this.buildQueue(); } - this.inProgress = new Set(queue.map((item) => item.filename)); - - for (const test of queue) { - const filename = test.filename; - const content = test.getContent(); - const meta = test.title = this.getMeta(content); - - const absolutePath = test.getAbsolutePath(); - const context = this.generateContext(test); - const relativePath = test.getRelativePath(); - const code = this.mergeScripts(relativePath, meta, content); - try { - vm.runInContext(code, context, { - filename: absolutePath - }); - } catch (err) { - this.fail(filename, { - name: '', - message: err.message, - stack: inspect(err) - }, 'UNCAUGHT'); - this.inProgress.delete(filename); + this.inProgress = new Set(queue.map((spec) => spec.filename)); + + for (const spec of queue) { + const testFileName = spec.filename; + const content = spec.getContent(); + const meta = spec.title = this.getMeta(content); + + const absolutePath = spec.getAbsolutePath(); + const context = this.generateContext(spec); + const relativePath = spec.getRelativePath(); + const scriptsToRun = []; + // Scripts specified with the `// META: script=` header + if (meta.script) { + for (const script of meta.script) { + scriptsToRun.push({ + filename: this.resource.toRealFilePath(relativePath, script), + code: this.resource.read(relativePath, script, false) + }); + } + } + // The actual test + scriptsToRun.push({ + code: content, + filename: absolutePath + }); + + for (const { code, filename } of scriptsToRun) { + try { + vm.runInContext(code, context, { filename }); + } catch (err) { + this.fail( + testFileName, + { + status: NODE_UNCAUGHT, + name: 'evaluation in WPTRunner.runJsTests()', + message: err.message, + stack: inspect(err) + }, + kUncaught + ); + this.inProgress.delete(filename); + break; + } } } - this.tryFinish(); + + process.on('exit', () => { + const total = this.specMap.size; + if (this.inProgress.size > 0) { + for (const filename of this.inProgress) { + this.fail(filename, { name: 'Unknown' }, kIncomplete); + } + } + inspect.defaultOptions.depth = Infinity; + console.log(this.results); + + const failures = []; + let expectedFailures = 0; + let skipped = 0; + for (const key of Object.keys(this.results)) { + const item = this.results[key]; + if (item.fail && item.fail.unexpected) { + failures.push(key); + } + if (item.fail && item.fail.expected) { + expectedFailures++; + } + if (item.skip) { + skipped++; + } + } + const ran = total - skipped; + const passed = ran - expectedFailures - failures.length; + console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`, + `${passed} passed, ${expectedFailures} expected failures,`, + `${failures.length} unexpected failures`); + if (failures.length > 0) { + const file = path.join('test', 'wpt', 'status', `${this.path}.json`); + throw new Error( + `Found ${failures.length} unexpected failures. ` + + `Consider updating ${file} for these files:\n${failures.join('\n')}`); + } + }); } mock(testfile) { @@ -410,115 +478,124 @@ class WPTRunner { sandbox.self = sandbox; // TODO(joyeecheung): we are not a window - work with the upstream to // add a new scope for us. - return context; } - resultCallback(filename, test) { - switch (test.status) { + getTestTitle(filename) { + const spec = this.specMap.get(filename); + const title = spec.meta && spec.meta.title; + return title ? `${filename} : ${title}` : filename; + } + + // Map WPT test status to strings + getTestStatus(status) { + switch (status) { case 1: - this.fail(filename, test, 'FAILURE'); - break; + return kFail; case 2: - this.fail(filename, test, 'TIMEOUT'); - break; + return kTimeout; case 3: - this.fail(filename, test, 'INCOMPLETE'); - break; + return kIncomplete; + case NODE_UNCAUGHT: + return kUncaught; default: - this.succeed(filename, test); + return kPass; } } + /** + * Report the status of each specific test case (there could be multiple + * in one test file). + * + * @param {string} filename + * @param {Test} test The Test object returned by WPT harness + */ + resultCallback(filename, test) { + const status = this.getTestStatus(test.status); + const title = this.getTestTitle(filename); + console.log(`---- ${title} ----`); + if (status !== kPass) { + this.fail(filename, test, status); + } else { + this.succeed(filename, test, status); + } + } + + /** + * Report the status of each WPT test (one per file) + * + * @param {string} filename + * @param {Test[]} test The Test objects returned by WPT harness + */ completionCallback(filename, tests, harnessStatus) { + // Treat it like a test case failure if (harnessStatus.status === 2) { - assert.fail(`test harness timed out in ${filename}`); + const title = this.getTestTitle(filename); + console.log(`---- ${title} ----`); + this.resultCallback(filename, { status: 2, name: 'Unknown' }); } this.inProgress.delete(filename); - this.tryFinish(); } - tryFinish() { - if (this.inProgress.size > 0) { - return; + addTestResult(filename, item) { + let result = this.results[filename]; + if (!result) { + result = this.results[filename] = {}; } - - this.reportResults(); - } - - reportResults() { - const unexpectedFailures = []; - for (const [filename, items] of this.results) { - const test = this.tests.get(filename); - let title = test.meta && test.meta.title; - title = title ? `${filename} : ${title}` : filename; - console.log(`---- ${title} ----`); - for (const item of items) { - switch (item.type) { - case FAILED: { - if (test.failReasons.length) { - console.log(`[EXPECTED_FAILURE] ${item.test.name}`); - console.log(test.failReasons.join('; ')); - } else { - console.log(`[UNEXPECTED_FAILURE] ${item.test.name}`); - unexpectedFailures.push([title, filename, item]); - } - break; - } - case PASSED: { - console.log(`[PASSED] ${item.test.name}`); - break; - } - case SKIPPED: { - console.log(`[SKIPPED] ${item.reason}`); - break; - } - } + if (item.status === kSkip) { + // { filename: { skip: 'reason' } } + result[kSkip] = item.reason; + } else { + // { filename: { fail: { expected: [ ... ], + // unexpected: [ ... ] } }} + if (!result[item.status]) { + result[item.status] = {}; } - } - - if (unexpectedFailures.length > 0) { - for (const [title, filename, item] of unexpectedFailures) { - console.log(`---- ${title} ----`); - console.log(`[${item.reason}] ${item.test.name}`); - console.log(item.test.message); - console.log(item.test.stack); - const command = `${process.execPath} ${process.execArgv}` + - ` ${require.main.filename} ${filename}`; - console.log(`Command: ${command}\n`); + const key = item.expected ? 'expected' : 'unexpected'; + if (!result[item.status][key]) { + result[item.status][key] = []; + } + if (result[item.status][key].indexOf(item.reason) === -1) { + result[item.status][key].push(item.reason); } - assert.fail(`${unexpectedFailures.length} unexpected failures found`); - } - } - - addResult(filename, item) { - const result = this.results.get(filename); - if (result) { - result.push(item); - } else { - this.results.set(filename, [item]); } } - succeed(filename, test) { - this.addResult(filename, { - type: PASSED, - test - }); + succeed(filename, test, status) { + console.log(`[${status.toUpperCase()}] ${test.name}`); } - fail(filename, test, reason) { - this.addResult(filename, { - type: FAILED, - test, - reason + fail(filename, test, status) { + const spec = this.specMap.get(filename); + const expected = !!(spec.failReasons.length); + if (expected) { + console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`); + console.log(spec.failReasons.join('; ')); + } else { + console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`); + } + if (status === kFail || status === kUncaught) { + console.log(test.message); + console.log(test.stack); + } + const command = `${process.execPath} ${process.execArgv}` + + ` ${require.main.filename} ${filename}`; + console.log(`Command: ${command}\n`); + this.addTestResult(filename, { + expected, + status: kFail, + reason: test.message || status }); } skip(filename, reasons) { - this.addResult(filename, { - type: SKIPPED, - reason: reasons.join('; ') + const title = this.getTestTitle(filename); + console.log(`---- ${title} ----`); + const joinedReasons = reasons.join('; '); + console.log(`[SKIPPED] ${joinedReasons}`); + this.addTestResult(filename, { + status: kSkip, + reason: joinedReasons }); } @@ -546,36 +623,22 @@ class WPTRunner { } } - mergeScripts(base, meta, content) { - if (!meta.script) { - return content; - } - - // only one script - let result = ''; - for (const script of meta.script) { - result += this.resource.read(base, script, false); - } - - return result + content; - } - buildQueue() { const queue = []; - for (const test of this.tests.values()) { - const filename = test.filename; - if (test.skipReasons.length > 0) { - this.skip(filename, test.skipReasons); + for (const spec of this.specMap.values()) { + const filename = spec.filename; + if (spec.skipReasons.length > 0) { + this.skip(filename, spec.skipReasons); continue; } - const lackingIntl = intlRequirements.isLacking(test.requires); + const lackingIntl = intlRequirements.isLacking(spec.requires); if (lackingIntl) { this.skip(filename, [ `requires ${lackingIntl}` ]); continue; } - queue.push(test); + queue.push(spec); } return queue; }