test: allow test names in WPT Status expected failures

nodejs · Jun 17, 2022 · 5f255de · 5f255de
1 parent 2804b84
commit 5f255de
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 7 deletions.
diff --git a/test/common/wpt.js b/test/common/wpt.js
@@ -152,7 +152,7 @@ class WPTTestSpec {
     this.filename = filename;
 
     this.requires = new Set();
-    this.failReasons = [];
+    this.failedTests = [];
     this.skipReasons = [];
     for (const item of rules) {
       if (item.requires.length) {
@@ -161,7 +161,7 @@ class WPTTestSpec {
         }
       }
       if (Array.isArray(item.fail?.expected)) {
-        this.failReasons.push(...item.fail.expected);
+        this.failedTests.push(...item.fail.expected);
       }
       if (item.skip) {
         this.skipReasons.push(item.skip);
@@ -584,7 +584,9 @@ class WPTRunner {
       if (!result[item.status][key]) {
         result[item.status][key] = [];
       }
-      if (result[item.status][key].indexOf(item.reason) === -1) {
+      const hasName = result[item.status][key].includes(item.name)
+      const hasReason = result[item.status][key].includes(item.reason)
+      if (!hasName && !hasReason) {
         result[item.status][key].push(item.reason);
       }
     }
@@ -596,7 +598,7 @@ class WPTRunner {
 
   fail(filename, test, status) {
     const spec = this.specMap.get(filename);
-    const expected = spec.failReasons.includes(test.message || status);
+    const expected = spec.failedTests.includes(test.name) || spec.failedTests.includes(test.message || status);
     if (expected) {
       console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
       console.log(test.message || status);
@@ -611,6 +613,7 @@ class WPTRunner {
                     ` ${require.main.filename} ${filename}`;
     console.log(`Command: ${command}\n`);
     this.addTestResult(filename, {
+      name: test.name,
       expected,
       status: kFail,
       reason: test.message || status

diff --git a/test/wpt/README.md b/test/wpt/README.md
@@ -93,7 +93,7 @@ add this to `test/wpt/status/url.json`:
   "url-searchparams.any.js": {
     "fail": {
       "expected": [
-        "exhibited error message"
+        "test name"
       ]
     }
   }
@@ -162,7 +162,9 @@ expected failures.
     // Optional: tests failing with this message are expected
     "fail": {
       "expected": [
-        "exhibited error message"
+        "exhibited error message",
+        // or
+        "test name"
       ]
     }
   }

diff --git a/test/wpt/status/html/webappapis/atob.json b/test/wpt/status/html/webappapis/atob.json
@@ -2,7 +2,7 @@
   "base64.any.js": {
     "fail": {
       "expected": [
-        "promise_test: Unhandled rejection with value: object \"Error: ENOENT: no such file or directory, open '/Users/panva/repo/node/test/fixtures/wpt/fetch/data-urls/resources/base64.json'\""
+        "atob() setup."
       ]
     }
   }