diff --git a/test/common/wpt.js b/test/common/wpt.js index d8333c19c60014..99741246e90355 100644 --- a/test/common/wpt.js +++ b/test/common/wpt.js @@ -152,7 +152,7 @@ class WPTTestSpec { this.filename = filename; this.requires = new Set(); - this.failReasons = []; + this.failedTests = []; this.skipReasons = []; for (const item of rules) { if (item.requires.length) { @@ -161,7 +161,7 @@ class WPTTestSpec { } } if (Array.isArray(item.fail?.expected)) { - this.failReasons.push(...item.fail.expected); + this.failedTests.push(...item.fail.expected); } if (item.skip) { this.skipReasons.push(item.skip); @@ -584,7 +584,9 @@ class WPTRunner { if (!result[item.status][key]) { result[item.status][key] = []; } - if (result[item.status][key].indexOf(item.reason) === -1) { + const hasName = result[item.status][key].includes(item.name) + const hasReason = result[item.status][key].includes(item.reason) + if (!hasName && !hasReason) { result[item.status][key].push(item.reason); } } @@ -596,7 +598,7 @@ class WPTRunner { fail(filename, test, status) { const spec = this.specMap.get(filename); - const expected = spec.failReasons.includes(test.message || status); + const expected = spec.failedTests.includes(test.name) || spec.failedTests.includes(test.message || status); if (expected) { console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`); console.log(test.message || status); @@ -611,6 +613,7 @@ class WPTRunner { ` ${require.main.filename} ${filename}`; console.log(`Command: ${command}\n`); this.addTestResult(filename, { + name: test.name, expected, status: kFail, reason: test.message || status diff --git a/test/wpt/README.md b/test/wpt/README.md index 0bb43be81703e5..bafea19be3cc6c 100644 --- a/test/wpt/README.md +++ b/test/wpt/README.md @@ -93,7 +93,7 @@ add this to `test/wpt/status/url.json`: "url-searchparams.any.js": { "fail": { "expected": [ - "exhibited error message" + "test name" ] } } @@ -162,7 +162,9 @@ expected failures. // Optional: tests failing with this message are expected "fail": { "expected": [ - "exhibited error message" + "exhibited error message", + // or + "test name" ] } } diff --git a/test/wpt/status/html/webappapis/atob.json b/test/wpt/status/html/webappapis/atob.json index 0a8ec46489452d..4d8c4df8e234e3 100644 --- a/test/wpt/status/html/webappapis/atob.json +++ b/test/wpt/status/html/webappapis/atob.json @@ -2,7 +2,7 @@ "base64.any.js": { "fail": { "expected": [ - "promise_test: Unhandled rejection with value: object \"Error: ENOENT: no such file or directory, open '/Users/panva/repo/node/test/fixtures/wpt/fetch/data-urls/resources/base64.json'\"" + "atob() setup." ] } }