From 54261f32940dd13c7b458a123ad671b1fb3527a1 Mon Sep 17 00:00:00 2001 From: Filip Skokan Date: Fri, 31 Mar 2023 19:56:40 +0200 Subject: [PATCH] test: run WPT files in parallel again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-URL: https://github.com/nodejs/node/pull/47283 Refs: https://github.com/nodejs/node/issues/47146 Reviewed-By: Michaël Zasso Reviewed-By: Yagiz Nizipli --- Makefile | 1 + test/common/wpt.js | 83 +++++++++++++++++++++++-------------- test/wpt/testcfg.py | 2 +- tools/merge-wpt-reports.mjs | 32 ++++++++++++++ 4 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 tools/merge-wpt-reports.mjs diff --git a/Makefile b/Makefile index 0be0659d372d2d..d8a5b88a5235b1 100644 --- a/Makefile +++ b/Makefile @@ -603,6 +603,7 @@ test-wpt-report: $(RM) -r out/wpt mkdir -p out/wpt WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt + $(NODE) "$$PWD/tools/merge-wpt-reports.mjs" .PHONY: test-simple test-simple: | cctest # Depends on 'all'. diff --git a/test/common/wpt.js b/test/common/wpt.js index 57c409f5cc8a0f..342324182138ea 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -10,6 +10,8 @@ const os = require('os'); const { inspect } = require('util'); const { Worker } = require('worker_threads'); +const workerPath = path.join(__dirname, 'wpt/worker.js'); + function getBrowserProperties() { const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481 const release = /^\d+\.\d+\.\d+$/.test(version); @@ -57,7 +59,8 @@ function codeUnitStr(char) { } class WPTReport { - constructor() { + constructor(path) { + this.filename = `report-${path.replaceAll('/', '-')}.json`; this.results = []; this.time_start = Date.now(); } @@ -96,26 +99,18 @@ class WPTReport { return result; }); - if (fs.existsSync('out/wpt/wptreport.json')) { - const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json')); - this.results = [...prev.results, ...this.results]; - this.time_start = prev.time_start; - this.time_end = Math.max(this.time_end, prev.time_end); - this.run_info = prev.run_info; - } else { - /** - * Return required and some optional properties - * https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335 - */ - this.run_info = { - product: 'node.js', - ...getBrowserProperties(), - revision: process.env.WPT_REVISION || 'unknown', - os: getOs(), - }; - } + /** + * Return required and some optional properties + * https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335 + */ + this.run_info = { + product: 'node.js', + ...getBrowserProperties(), + revision: process.env.WPT_REVISION || 'unknown', + os: getOs(), + }; - fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this)); + fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify(this)); } } @@ -402,6 +397,29 @@ const kIncomplete = 'incomplete'; const kUncaught = 'uncaught'; const NODE_UNCAUGHT = 100; +const limit = (concurrency) => { + let running = 0; + const queue = []; + + const execute = async (fn) => { + if (running < concurrency) { + running++; + try { + await fn(); + } finally { + running--; + if (queue.length > 0) { + execute(queue.shift()); + } + } + } else { + queue.push(fn); + } + }; + + return execute; +}; + class WPTRunner { constructor(path) { this.path = path; @@ -425,7 +443,7 @@ class WPTRunner { this.scriptsModifier = null; if (process.env.WPT_REPORT != null) { - this.report = new WPTReport(); + this.report = new WPTReport(path); } } @@ -543,6 +561,8 @@ class WPTRunner { this.inProgress = new Set(queue.map((spec) => spec.filename)); + const run = limit(os.availableParallelism()); + for (const spec of queue) { const testFileName = spec.filename; const content = spec.getContent(); @@ -576,15 +596,7 @@ class WPTRunner { this.scriptsModifier?.(obj); scriptsToRun.push(obj); - /** - * Example test with no META variant - * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4 - * - * Example test with multiple META variants - * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9 - */ - for (const variant of meta.variant || ['']) { - const workerPath = path.join(__dirname, 'wpt/worker.js'); + const runWorker = async (variant) => { const worker = new Worker(workerPath, { execArgv: this.flags, workerData: { @@ -635,6 +647,17 @@ class WPTRunner { }); await events.once(worker, 'exit').catch(() => {}); + }; + + /** + * Example test with no META variant + * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4 + * + * Example test with multiple META variants + * https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9 + */ + for (const variant of meta.variant || ['']) { + run(() => runWorker(variant)); } } diff --git a/test/wpt/testcfg.py b/test/wpt/testcfg.py index 3c356cf474d83c..db235699ddfe57 100644 --- a/test/wpt/testcfg.py +++ b/test/wpt/testcfg.py @@ -3,4 +3,4 @@ import testpy def GetConfiguration(context, root): - return testpy.SimpleTestConfiguration(context, root, 'wpt') + return testpy.ParallelTestConfiguration(context, root, 'wpt') diff --git a/tools/merge-wpt-reports.mjs b/tools/merge-wpt-reports.mjs new file mode 100644 index 00000000000000..9199714f4b1032 --- /dev/null +++ b/tools/merge-wpt-reports.mjs @@ -0,0 +1,32 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as url from 'node:url'; + +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +const outDir = path.resolve(__dirname, '../out/wpt'); +const files = fs.readdirSync(outDir); +const reports = files.filter((file) => file.endsWith('.json')); + +const startTimes = []; +const endTimes = []; +const results = []; +let runInfo; + +for (const file of reports) { + const report = JSON.parse(fs.readFileSync(path.resolve(outDir, file))); + fs.unlinkSync(path.resolve(outDir, file)); + results.push(...report.results); + startTimes.push(report.time_start); + endTimes.push(report.time_end); + runInfo ||= report.run_info; +} + +const mergedReport = { + time_start: Math.min(...startTimes), + time_end: Math.max(...endTimes), + run_info: runInfo, + results, +}; + +fs.writeFileSync(path.resolve(outDir, 'wptreport.json'), JSON.stringify(mergedReport));