diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 363a14ce9ea31..51a841f8cae4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,26 +173,12 @@ jobs: run: npm run build:dev - name: Test types run: npm run test:types - # On Linux we run all Chrome tests without retries and Firefox tests with retries. - - name: Run all Chrome tests (only on Linux) + - name: Run all tests with xvfb if: ${{ matrix.spec.name == 'Linux' }} - run: xvfb-run --auto-servernum npm run test:chrome - - name: Run all Firefox tests (only on Linux) - if: ${{ matrix.spec.name == 'Linux' }} - uses: nick-invision/retry@v2 - with: - command: xvfb-run --auto-servernum npm run test:firefox - timeout_minutes: 20 - max_attempts: 3 - # On other platforms we only run chrome:headless tests and continue on Firefox errors. - - name: Test Chrome Headless (not on Linux) - id: test-chrome - if: ${{ matrix.spec.name != 'Linux' }} - run: npm run test:chrome:headless - - name: Run all Firefox tests (not on Linux) + run: xvfb-run --auto-servernum npm run test + - name: Run all tests without xvfb if: ${{ matrix.spec.name != 'Linux' }} - continue-on-error: true - run: npm run test:firefox + run: npm run test - name: Test bundling and installation if: ${{ matrix.spec.name == 'Linux' }} run: | diff --git a/.mocharc.cjs b/.mocharc.cjs index e7168715d898e..30bf43695b9c8 100644 --- a/.mocharc.cjs +++ b/.mocharc.cjs @@ -22,6 +22,6 @@ module.exports = { exit: !!process.env.CI, retries: process.env.CI ? 2 : 0, parallel: !!process.env.PARALLEL, - timeout: 25 * 1000, + timeout: 25_000, reporter: process.env.CI ? 'spec' : 'dot', }; diff --git a/package-lock.json b/package-lock.json index 70e3a1267fae3..d09af77e4c128 100644 --- a/package-lock.json +++ b/package-lock.json @@ -80,7 +80,8 @@ "text-diff": "1.0.1", "tsd": "0.22.0", "tsx": "3.8.2", - "typescript": "4.7.4" + "typescript": "4.7.4", + "zod": "3.18.0" }, "engines": { "node": ">=14.1.0" @@ -7809,6 +7810,15 @@ "optionalDependencies": { "commander": "^2.20.3" } + }, + "node_modules/zod": { + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.18.0.tgz", + "integrity": "sha512-gwTm8RfUCe8l9rDwN5r2A17DkAa8Ez4Yl4yXqc5VqeGaXaJahzYYXbTwvhroZi0SNBqTwh/bKm2N0mpCzuw4bA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } }, "dependencies": { @@ -13501,6 +13511,12 @@ "lodash.isequal": "^4.5.0", "validator": "^13.7.0" } + }, + "zod": { + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.18.0.tgz", + "integrity": "sha512-gwTm8RfUCe8l9rDwN5r2A17DkAa8Ez4Yl4yXqc5VqeGaXaJahzYYXbTwvhroZi0SNBqTwh/bKm2N0mpCzuw4bA==", + "dev": true } } } diff --git a/package.json b/package.json index ef1bf03e1bde2..f6e9156baafb0 100644 --- a/package.json +++ b/package.json @@ -27,14 +27,14 @@ "node": ">=14.1.0" }, "scripts": { - "test": "c8 --check-coverage --lines 93 run-s test:chrome:* test:firefox", + "test": "cross-env MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 c8 --check-coverage --lines 93 node utils/mochaRunner/lib/main.js", "test:types": "tsd", "test:install": "scripts/test-install.sh", - "test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", + "test:firefox": "npm run test -- --test-suite firefox-headless", "test:chrome": "run-s test:chrome:*", - "test:chrome:headless": "cross-env HEADLESS=true PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", - "test:chrome:headless-chrome": "cross-env HEADLESS=chrome PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", - "test:chrome:headful": "cross-env HEADLESS=false PUPPETEER_DEFERRED_PROMISE_DEBUG_TIMEOUT=20000 mocha", + "test:chrome:headless": "npm run test -- --test-suite chrome-headless", + "test:chrome:headless-chrome": "npm run test -- --test-suite chrome-new-headless", + "test:chrome:headful": "npm run test -- --test-suite chrome-headful", "prepublishOnly": "npm run build", "prepare": "node typescript-if-required.js && husky install", "lint": "run-s lint:prettier lint:eslint", @@ -139,6 +139,7 @@ "text-diff": "1.0.1", "tsd": "0.22.0", "tsx": "3.8.2", - "typescript": "4.7.4" + "typescript": "4.7.4", + "zod": "3.18.0" } } diff --git a/test/TestExpectations.json b/test/TestExpectations.json new file mode 100644 index 0000000000000..4983051534ce9 --- /dev/null +++ b/test/TestExpectations.json @@ -0,0 +1,1946 @@ +[ + { + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ariaqueryhandler.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should fire target events", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should isolate localStorage and cookies", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should wait for a target", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext should work across sessions", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[browsercontext.spec] BrowserContext window.open should use parent tab context", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[CDPSession.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[chromiumonly.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click on checkbox label and toggle", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click the button if window.Node is removed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[click.spec] Page.click should click with disabled javascript", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.cookies should get cookies from multiple urls", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.deleteCookie should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should default to setting secure cookie for HTTPS websites", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should isolate cookies in browser contexts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie on a different domain", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set a cookie with a path", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookie with reasonable defaults", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set cookies from a frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set multiple cookies", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should set secure same-site cookies from a frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[cookies.spec] Cookie specs Page.setCookie should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[coverage.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.deleteCookie() should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[defaultbrowsercontext.spec] DefaultBrowserContext page.setCookie() should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[dialog.spec] Page.Events.Dialog should allow accepting prompts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[drag-and-drop.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should handle nested frames", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should handle nested frames", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should return null for invisible elements", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should return null for invisible elements", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boundingBox should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should return null for invisible elements", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.boxModel should return null for invisible elements", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.click should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[elementhandle.spec] ElementHandle specs ElementHandle.contentFrame should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulate should support clicking", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateCPUThrottling should change the CPU throttling rate successfully", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateMediaFeatures should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateMediaType should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateNetworkConditions should change navigator.connection.effectiveType", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should throw for invalid timezone IDs", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should throw for invalid timezone IDs", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateTimezone should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateVisionDeficiency should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.emulateVisionDeficiency should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[emulation.spec] Emulation Page.viewport should support landscape emulation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should be able to throw a tricky error", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should fail for circular object", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should not throw an error when evaluation does a navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should return undefined for non-serializable objects", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should return undefined for objects with symbols", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should simulate a user gesture", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw a nice error after a navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw if elementHandles are from other frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should throw when evaluation triggers reload", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluate should work from-inside an exposed function", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should evaluate before anything else on the page", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should evaluate before anything else on the page", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should work with CSP", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[evaluation.spec] Evaluation specs Page.evaluateOnNewDocument should work with CSP", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["linux"], + "parameters": ["chrome", "headful"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[fixtures.spec] Fixtures dumpio option should work with pipe option", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should handle nested frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report different frame instance when frame re-attaches", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame from-inside shadow DOM", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.name()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should report frame.parent()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should send events when frames are manipulated dynamically", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame Management should support lazy frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame.evaluate should throw for detached frames", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[frame.spec] Frame specs Frame.executionContext should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[headful.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_override.spec] Emulate idle state changing idle state emulation causes change of the IdleDetector state", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_override.spec] Emulate idle state changing idle state emulation causes change of the IdleDetector state", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[idle_overrides.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttperrors.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with mixed content", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors should work with request interception", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[ignorehttpserrors.spec] ignoreHTTPSErrors Response.securityDetails Network redirects should report SecurityDetails", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[input.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.click should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.jsonValue should not work with dates", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandle.jsonValue should not work with dates", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[jshandle.spec] JSHandle JSHandleonValue should not work with dates", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard ElementHandle.press should support |text| option", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should press the meta key", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should press the metaKey", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should report shiftKey", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should send a character with sendCharacter", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should specify location", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should specify repeat property", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type all kinds of characters", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type emoji", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[keyboard.spec] Keyboard should type emoji into an iframe", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Browser target events should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Browser.Events.disconnected should be emitted when: browser gets closed, disconnected or underlying websocket gets closed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to connect to the same page simultaneously", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to reconnect to a disconnected browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should support targetFilter option", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable does not exist does not return /usr/bin/chromium-browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.executablePath when the product is chrome, platform is not darwin, and arch is arm64 and the executable exists returns /usr/bin/chromium-browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should filter out ignored default arguments in Chrome", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should have custom URL when launching browser", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch should launch Chrome properly with --no-startup-window and waitForInitialPage=false", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch tmp profile should be cleaned up", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.launch userDataDir argument with non-existent dir", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should send mouse wheel events", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should trigger hover state", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should trigger hover state with removed window.Node", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[mouse.spec] Mouse should tween mouse movement", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should navigate subframes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should navigate subframes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should reject when frame detaches", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should reject when frame detaches", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.goto should return matching responses", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should fail when frame detaches", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should fail when frame detaches", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Frame.waitForNavigation should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goBack should work with HistoryAPI", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad SSL", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when navigating to bad url", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should fail when server returns 204", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to dataURL and fire dataURL requests", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle0", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to empty page with networkidle2", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should navigate to URL with hash and fire requests without hash", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should send referer", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should wait for network idle to succeed navigation", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should work when navigating to data url", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.goto should work with subframes return 204", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work when subframe issues window.stop()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with DOM history.back()/history.forward()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.pushState()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[navigation.spec] navigation Page.waitForNavigation should work with history.replaceState()", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network \"after each\" hook for \"should wait until response completes\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.Request", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.Response", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFailed", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestFinished", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events Page.Events.RequestServedFromCache", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events should fire events in proper order", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Network Events should support redirects", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should allow disable authentication", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should fail if wrong credentials", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should not disable caching", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.authenticate should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Page.setExtraHTTPHeaders should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network raw network headers Cross-origin set-cookie", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["FAIL", "PASS"] + }, + { + "testIdPattern": "[network.spec] network raw network headers Same-origin set-cookie subresource", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.frame should work for subframe navigation request", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.headers should define Chrome as user agent header", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.headers should define Firefox as user agent header", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.initiator should return the initiator", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.initiator should return the initiator", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work when navigating to image", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.isNavigationRequest should work with request interception", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.postData should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Request.postData should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should throw if the response does not have a body", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.buffer should work with compression", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromCache should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromCache should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.fromServiceWorker Response.fromServiceWorker", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.json should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should return uncompressed text", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should return uncompressed text", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should throw when requesting body of redirected response", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should throw when requesting body of redirected response", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should wait until response completes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should wait until response completes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.text should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[network.spec] network Response.timing returns timing information", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF \"after all\" hook for \"should keep track of a frames OOP state\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF \"before all\" hook for \"should keep track of a frames OOP state\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should support OOP iframes becoming normal iframes again", + "platforms": ["darwin"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should keep track of a frames OOP state", + "platforms": ["darwin"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[oopif.spec] OOPIF should wait for inner OOPIFs", + "platforms": ["linux"], + "parameters": ["chrome", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[launcher.spec] Launcher specs Puppeteer Puppeteer.connect should be able to connect to a browser with no page targets", + "platforms": ["linux", "darwin", "win32"], + "parameters": ["chrome", "chrome-headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[page.spec] Page \"before each\" hook for \"should return the page title\"", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox", "headless"], + "expectations": ["PASS", "FAIL"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should deny permission when not listed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should grant permission when listed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should grant persistent-storage", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should isolate permissions between browser contexts", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should reset permissions", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page BrowserContext.overridePermissions should trigger permission onchange", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page ExecutionContext.queryObjects should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page ExecutionContext.queryObjects should work for non-blank page", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.addStyleTag should throw when added with content to the CSP page", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should *not* run beforeunload by default", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should run beforeunload if asked for", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.close should terminate network waiters", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Close should work with window.close", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location and stack trace for console API calls", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location when fetch fails", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should have location when fetch fails", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should not fail for window object", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should not fail for window object", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should trigger correct Log", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should trigger correct Log", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Console should work for different console API calls", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.error should throw when page crashes", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and rel=noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and rel=noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and with rel=opener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and with rel=opener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and without rel=opener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with clicking target=_blank and without rel=opener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with fake-clicking target=_blank and rel=noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with noopener", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.Events.Popup should work with noopener", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should await returned promise", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should await returned promise", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should be callable from-inside evaluateOnNewDocument", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should be callable from-inside evaluateOnNewDocument", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should fallback to default export when passed a module object", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should fallback to default export when passed a module object", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should not throw when frames detach", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should not throw when frames detach", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should support throwing \"null\"", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should support throwing \"null\"", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should survive navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should survive navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should throw exception in page context", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should throw exception in page context", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames before navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work on frames before navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work with complex objects", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.exposeFunction should work with complex objects", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics metrics event fired on console.timeStamp", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics metrics event fired on console.timeStamp", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics should get metrics from a page", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.metrics should get metrics from a page", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.select should work when re-defining top-level Event class", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass after cross-process navigation", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass after cross-process navigation", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP header", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP header", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP in iframes as well", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP in iframes as well", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP meta tag", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setBypassCSP should bypass CSP meta tag", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should enable or disable the cache based on the state passed", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should enable or disable the cache based on the state passed", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setCacheEnabled should stay disabled when toggling request interception on/off", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setGeolocation should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setJavaScriptEnabled should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setJavaScriptEnabled should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should emulate navigator.onLine", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", + "platforms": ["darwin"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setOfflineMode should work", + "platforms": ["linux"], + "parameters": ["firefox", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[page.spec] Page Page.setUserAgent should work with additional userAgentMetdata", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[proxy.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[proxy.spec] request proxy in incognito browser context should proxy requests when configured at context level", + "platforms": ["win32"], + "parameters": ["chrome", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[requestinterception-experimental.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[requestinterception.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should capture full element when larger than viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should use scale for clip", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should fail to screenshot a detached element", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work for an element with an offset", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a null viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots ElementHandle.screenshot should work with a rotated element", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should allow transparency", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should clip rect", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should get screenshot bigger than the viewport", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should render white background on jpeg file", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should return base64", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should take fullPage screenshots", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work in \"fromSurface: false\" mode", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome", "headless"], + "expectations": ["FAIL"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work in \"fromSurface: false\" mode", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["chrome", "headless"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[screenshot.spec] Screenshots Page.screenshot should work with webp", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target Browser.waitForTarget should wait for a target", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should be able to use async waitForTarget", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should create a worker from a service worker", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should create a worker from a shared worker", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should have an opener", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should not crash while redirecting if original request was missed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should not report uninitialized pages", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a new page is created and closed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a service worker is created and destroyed", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[target.spec] Target should report when a target url changes", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[TargetManager.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[touchscreen.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[tracing.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForFunction should work with strict CSP policy", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector Page.waitForSelector is shortcut for main frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should run in specified frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should throw when frame is detached", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForSelector should work with removed MutationObserver", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForXPath should run in specified frame", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[waittask.spec] waittask specs Frame.waitForXPath should throw when frame is detached", + "platforms": ["darwin", "linux", "win32"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + }, + { + "testIdPattern": "[worker.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + } +] diff --git a/test/TestSuites.json b/test/TestSuites.json new file mode 100644 index 0000000000000..43d02dd09ef85 --- /dev/null +++ b/test/TestSuites.json @@ -0,0 +1,41 @@ +{ + "testSuites": [ + { + "id": "chrome-headless", + "platforms": ["linux", "win32", "darwin"], + "parameters": ["chrome", "headless"] + }, + { + "id": "chrome-headful", + "platforms": ["linux"], + "parameters": ["chrome", "headful"] + }, + { + "id": "chrome-new-headless", + "platforms": ["linux"], + "parameters": ["chrome", "chrome-headless"] + }, + { + "id": "firefox-headless", + "platforms": ["linux"], + "parameters": ["firefox", "headless"] + } + ], + "parameterDefinitons": { + "chrome": { + "PUPPETEER_PRODUCT": "chrome" + }, + "firefox": { + "PUPPETEER_PRODUCT": "firefox" + }, + "headless": { + "HEADLESS": "true" + }, + "headful": { + "HEADLESS": "false" + }, + "chrome-headless": { + "HEADLESS": "chrome" + } + } +} diff --git a/test/src/CDPSession.spec.ts b/test/src/CDPSession.spec.ts index 920c31051cb33..a45ff2c11a61b 100644 --- a/test/src/CDPSession.spec.ts +++ b/test/src/CDPSession.spec.ts @@ -20,11 +20,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; import {isErrorLike} from '../../lib/cjs/puppeteer/util/ErrorLike.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Target.createCDPSession', function () { +describe('Target.createCDPSession', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/EventEmitter.spec.ts b/test/src/EventEmitter.spec.ts index 67e1ca610d827..b62e932e6e1fe 100644 --- a/test/src/EventEmitter.spec.ts +++ b/test/src/EventEmitter.spec.ts @@ -17,6 +17,7 @@ import {EventEmitter} from '../../lib/cjs/puppeteer/common/EventEmitter.js'; import sinon from 'sinon'; import expect from 'expect'; +import {it} from './mocha-utils.js'; describe('EventEmitter', () => { let emitter: EventEmitter; diff --git a/test/src/NetworkManager.spec.ts b/test/src/NetworkManager.spec.ts index cd8d647746ffd..f0d8ee795eb48 100644 --- a/test/src/NetworkManager.spec.ts +++ b/test/src/NetworkManager.spec.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import {describeChromeOnly} from './mocha-utils.js'; - import expect from 'expect'; import { NetworkManager, @@ -25,12 +23,13 @@ import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {EventEmitter} from '../../lib/cjs/puppeteer/common/EventEmitter.js'; import {Frame} from '../../lib/cjs/puppeteer/common/Frame.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; +import {it} from './mocha-utils.js'; class MockCDPSession extends EventEmitter { async send(): Promise {} } -describeChromeOnly('NetworkManager', () => { +describe('NetworkManager', () => { it('should process extra info on multiple redirects', async () => { const mockCDPSession = new MockCDPSession(); new NetworkManager(mockCDPSession, true, { diff --git a/test/src/TargetManager.spec.ts b/test/src/TargetManager.spec.ts index 175cde72f3272..8e39bb12d47d6 100644 --- a/test/src/TargetManager.spec.ts +++ b/test/src/TargetManager.spec.ts @@ -14,8 +14,9 @@ * limitations under the License. */ -import {describeChromeOnly, getTestState} from './mocha-utils'; // eslint-disable-line import/extensions +import {getTestState} from './mocha-utils'; // eslint-disable-line import/extensions import utils from './utils.js'; +import {it} from './mocha-utils.js'; import expect from 'expect'; @@ -24,7 +25,7 @@ import { BrowserContext, } from '../../lib/cjs/puppeteer/common/Browser.js'; -describeChromeOnly('TargetManager', () => { +describe('TargetManager', () => { /* We use a special browser for this test as we need the --site-per-process flag */ let browser: Browser; let context: BrowserContext; diff --git a/test/src/accessibility.spec.ts b/test/src/accessibility.spec.ts index fe066e9d01beb..412f5bedba33f 100644 --- a/test/src/accessibility.spec.ts +++ b/test/src/accessibility.spec.ts @@ -21,10 +21,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Accessibility', function () { +describe('Accessibility', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); @@ -346,7 +346,7 @@ describeFailsFirefox('Accessibility', function () { }); // Firefox does not support contenteditable="plaintext-only". - describeFailsFirefox('plaintext contenteditable', function () { + describe('plaintext contenteditable', function () { it('plain text field with role should not have children', async () => { const {page} = getTestState(); diff --git a/test/src/ariaqueryhandler.spec.ts b/test/src/ariaqueryhandler.spec.ts index 3c05349084074..a34831c3f85f6 100644 --- a/test/src/ariaqueryhandler.spec.ts +++ b/test/src/ariaqueryhandler.spec.ts @@ -19,14 +19,14 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; import {ElementHandle} from '../../lib/cjs/puppeteer/common/ElementHandle.js'; import utils from './utils.js'; import assert from 'assert'; +import {it} from './mocha-utils.js'; -describeChromeOnly('AriaQueryHandler', () => { +describe('AriaQueryHandler', () => { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/browser.spec.ts b/test/src/browser.spec.ts index 375b3dba934fa..1c06124cc5ec5 100644 --- a/test/src/browser.spec.ts +++ b/test/src/browser.spec.ts @@ -16,6 +16,7 @@ import expect from 'expect'; import {getTestState, setupTestBrowserHooks} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Browser specs', function () { setupTestBrowserHooks(); diff --git a/test/src/browsercontext.spec.ts b/test/src/browsercontext.spec.ts index d7d5d6adaa1e6..ad4d3683f75f7 100644 --- a/test/src/browsercontext.spec.ts +++ b/test/src/browsercontext.spec.ts @@ -15,12 +15,9 @@ */ import expect from 'expect'; -import { - getTestState, - itFailsFirefox, - setupTestBrowserHooks, -} from './mocha-utils.js'; +import {getTestState, setupTestBrowserHooks} from './mocha-utils.js'; import {waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; describe('BrowserContext', function () { setupTestBrowserHooks(); @@ -60,7 +57,7 @@ describe('BrowserContext', function () { await context.close(); expect((await browser.pages()).length).toBe(1); }); - itFailsFirefox('window.open should use parent tab context', async () => { + it('window.open should use parent tab context', async () => { const {browser, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -75,7 +72,7 @@ describe('BrowserContext', function () { expect(popupTarget.browserContext()).toBe(context); await context.close(); }); - itFailsFirefox('should fire target events', async () => { + it('should fire target events', async () => { const {browser, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -99,7 +96,7 @@ describe('BrowserContext', function () { ]); await context.close(); }); - itFailsFirefox('should wait for a target', async () => { + it('should wait for a target', async () => { const {browser, puppeteer, server} = getTestState(); const context = await browser.createIncognitoBrowserContext(); @@ -156,7 +153,7 @@ describe('BrowserContext', function () { await context.close(); }); - itFailsFirefox('should isolate localStorage and cookies', async () => { + it('should isolate localStorage and cookies', async () => { const {browser, server} = getTestState(); // Create two incognito contexts. @@ -216,7 +213,7 @@ describe('BrowserContext', function () { expect(browser.browserContexts().length).toBe(1); }); - itFailsFirefox('should work across sessions', async () => { + it('should work across sessions', async () => { const {browser, puppeteer} = getTestState(); expect(browser.browserContexts().length).toBe(1); diff --git a/test/src/chromiumonly.spec.ts b/test/src/chromiumonly.spec.ts index 5cb9cd3f2e44a..8bf9f00ef8104 100644 --- a/test/src/chromiumonly.spec.ts +++ b/test/src/chromiumonly.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Chromium-Specific Launcher tests', function () { +describe('Chromium-Specific Launcher tests', function () { describe('Puppeteer.launch |browserURL| option', function () { it('should be able to connect using browserUrl, with and without trailing slash', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); @@ -138,7 +138,7 @@ describeChromeOnly('Chromium-Specific Launcher tests', function () { }); }); -describeChromeOnly('Chromium-Specific Page Tests', function () { +describe('Chromium-Specific Page Tests', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('Page.setRequestInterception should work with intervention headers', async () => { diff --git a/test/src/click.spec.ts b/test/src/click.spec.ts index bea173712862e..5de40fef24cc8 100644 --- a/test/src/click.spec.ts +++ b/test/src/click.spec.ts @@ -19,9 +19,9 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - itFailsFirefox, } from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; describe('Page.click', function () { setupTestBrowserHooks(); @@ -52,24 +52,21 @@ describe('Page.click', function () { }) ).toBe(42); }); - itFailsFirefox( - 'should click the button if window.Node is removed', - async () => { - const {page, server} = getTestState(); + it('should click the button if window.Node is removed', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/button.html'); + await page.goto(server.PREFIX + '/input/button.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return delete window.Node; + }); + await page.click('button'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return delete window.Node; - }); - await page.click('button'); - expect( - await page.evaluate(() => { - return (globalThis as any).result; - }) - ).toBe('Clicked'); - } - ); + return (globalThis as any).result; + }) + ).toBe('Clicked'); + }); // @see https://github.com/puppeteer/puppeteer/issues/4281 it('should click on a span with an inline element inside', async () => { const {page} = getTestState(); @@ -110,7 +107,7 @@ describe('Page.click', function () { }) ).toBe('Clicked'); }); - itFailsFirefox('should click with disabled javascript', async () => { + it('should click with disabled javascript', async () => { const {page, server} = getTestState(); await page.setJavaScriptEnabled(false); @@ -240,7 +237,7 @@ describe('Page.click', function () { ).toBe(false); }); - itFailsFirefox('should click on checkbox label and toggle', async () => { + it('should click on checkbox label and toggle', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/checkbox.html'); diff --git a/test/src/cookies.spec.ts b/test/src/cookies.spec.ts index cd06ba7ad2c05..f493271319e29 100644 --- a/test/src/cookies.spec.ts +++ b/test/src/cookies.spec.ts @@ -19,8 +19,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Cookie specs', () => { setupTestBrowserHooks(); @@ -128,7 +128,7 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox('should get cookies from multiple urls', async () => { + it('should get cookies from multiple urls', async () => { const {page} = getTestState(); await page.setCookie( { @@ -184,7 +184,7 @@ describe('Cookie specs', () => { }); }); describe('Page.setCookie', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -198,7 +198,7 @@ describe('Cookie specs', () => { }) ).toEqual('password=123456'); }); - itFailsFirefox('should isolate cookies in browser contexts', async () => { + it('should isolate cookies in browser contexts', async () => { const {page, server, browser} = getTestState(); const anotherContext = await browser.createIncognitoBrowserContext(); @@ -220,7 +220,7 @@ describe('Cookie specs', () => { expect(cookies2[0]!.value).toBe('page2value'); await anotherContext.close(); }); - itFailsFirefox('should set multiple cookies', async () => { + it('should set multiple cookies', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -257,7 +257,7 @@ describe('Cookie specs', () => { expect(cookies[0]!.session).toBe(true); expect(cookies[0]!.expires).toBe(-1); }); - itFailsFirefox('should set cookie with reasonable defaults', async () => { + it('should set cookie with reasonable defaults', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -288,7 +288,7 @@ describe('Cookie specs', () => { ] ); }); - itFailsFirefox('should set a cookie with a path', async () => { + it('should set a cookie with a path', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/grid.html'); @@ -365,22 +365,19 @@ describe('Cookie specs', () => { 'At least one of the url and domain needs to be specified' ); }); - itFailsFirefox( - 'should default to setting secure cookie for HTTPS websites', - async () => { - const {page, server} = getTestState(); + it('should default to setting secure cookie for HTTPS websites', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - const SECURE_URL = 'https://example.com'; - await page.setCookie({ - url: SECURE_URL, - name: 'foo', - value: 'bar', - }); - const [cookie] = await page.cookies(SECURE_URL); - expect(cookie!.secure).toBe(true); - } - ); + await page.goto(server.EMPTY_PAGE); + const SECURE_URL = 'https://example.com'; + await page.setCookie({ + url: SECURE_URL, + name: 'foo', + value: 'bar', + }); + const [cookie] = await page.cookies(SECURE_URL); + expect(cookie!.secure).toBe(true); + }); it('should be able to set unsecure cookie for HTTP website', async () => { const {page, server} = getTestState(); @@ -394,7 +391,7 @@ describe('Cookie specs', () => { const [cookie] = await page.cookies(HTTP_URL); expect(cookie!.secure).toBe(false); }); - itFailsFirefox('should set a cookie on a different domain', async () => { + it('should set a cookie on a different domain', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -422,7 +419,7 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox('should set cookies from a frame', async () => { + it('should set cookies from a frame', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/grid.html'); @@ -482,71 +479,68 @@ describe('Cookie specs', () => { }, ]); }); - itFailsFirefox( - 'should set secure same-site cookies from a frame', - async () => { - const {httpsServer, puppeteer, defaultBrowserOptions} = getTestState(); + it('should set secure same-site cookies from a frame', async () => { + const {httpsServer, puppeteer, defaultBrowserOptions} = getTestState(); - const browser = await puppeteer.launch({ - ...defaultBrowserOptions, - ignoreHTTPSErrors: true, - }); + const browser = await puppeteer.launch({ + ...defaultBrowserOptions, + ignoreHTTPSErrors: true, + }); - const page = await browser.newPage(); + const page = await browser.newPage(); - try { - await page.goto(httpsServer.PREFIX + '/grid.html'); - await page.evaluate(src => { - let fulfill!: () => void; - const promise = new Promise(x => { - return (fulfill = x); - }); - const iframe = document.createElement('iframe'); - document.body.appendChild(iframe); - iframe.onload = fulfill; - iframe.src = src; - return promise; - }, httpsServer.CROSS_PROCESS_PREFIX); - await page.setCookie({ - name: '127-same-site-cookie', - value: 'best', - url: httpsServer.CROSS_PROCESS_PREFIX, - sameSite: 'None', + try { + await page.goto(httpsServer.PREFIX + '/grid.html'); + await page.evaluate(src => { + let fulfill!: () => void; + const promise = new Promise(x => { + return (fulfill = x); }); + const iframe = document.createElement('iframe'); + document.body.appendChild(iframe); + iframe.onload = fulfill; + iframe.src = src; + return promise; + }, httpsServer.CROSS_PROCESS_PREFIX); + await page.setCookie({ + name: '127-same-site-cookie', + value: 'best', + url: httpsServer.CROSS_PROCESS_PREFIX, + sameSite: 'None', + }); - expect(await page.frames()[1]!.evaluate('document.cookie')).toBe( - '127-same-site-cookie=best' - ); - expectCookieEquals( - await page.cookies(httpsServer.CROSS_PROCESS_PREFIX), - [ - { - name: '127-same-site-cookie', - value: 'best', - domain: '127.0.0.1', - path: '/', - sameParty: false, - expires: -1, - size: 24, - httpOnly: false, - sameSite: 'None', - secure: true, - session: true, - sourcePort: 443, - sourceScheme: 'Secure', - }, - ] - ); - } finally { - await page.close(); - await browser.close(); - } + expect(await page.frames()[1]!.evaluate('document.cookie')).toBe( + '127-same-site-cookie=best' + ); + expectCookieEquals( + await page.cookies(httpsServer.CROSS_PROCESS_PREFIX), + [ + { + name: '127-same-site-cookie', + value: 'best', + domain: '127.0.0.1', + path: '/', + sameParty: false, + expires: -1, + size: 24, + httpOnly: false, + sameSite: 'None', + secure: true, + session: true, + sourcePort: 443, + sourceScheme: 'Secure', + }, + ] + ); + } finally { + await page.close(); + await browser.close(); } - ); + }); }); describe('Page.deleteCookie', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); diff --git a/test/src/coverage.spec.ts b/test/src/coverage.spec.ts index 6e0fc5fe55f93..930f33cd4b01e 100644 --- a/test/src/coverage.spec.ts +++ b/test/src/coverage.spec.ts @@ -19,11 +19,11 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Coverage specs', function () { - describeChromeOnly('JSCoverage', function () { + describe('JSCoverage', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); @@ -202,7 +202,7 @@ describe('Coverage specs', function () { }); }); - describeChromeOnly('CSSCoverage', function () { + describe('CSSCoverage', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/defaultbrowsercontext.spec.ts b/test/src/defaultbrowsercontext.spec.ts index fea692932ead9..f9ff849b11907 100644 --- a/test/src/defaultbrowsercontext.spec.ts +++ b/test/src/defaultbrowsercontext.spec.ts @@ -19,8 +19,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('DefaultBrowserContext', function () { setupTestBrowserHooks(); @@ -48,7 +48,7 @@ describe('DefaultBrowserContext', function () { }, ]); }); - itFailsFirefox('page.setCookie() should work', async () => { + it('page.setCookie() should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -78,7 +78,7 @@ describe('DefaultBrowserContext', function () { }, ]); }); - itFailsFirefox('page.deleteCookie() should work', async () => { + it('page.deleteCookie() should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); diff --git a/test/src/dialog.spec.ts b/test/src/dialog.spec.ts index 27f337eec1b20..890437648ec03 100644 --- a/test/src/dialog.spec.ts +++ b/test/src/dialog.spec.ts @@ -20,8 +20,8 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - itFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Page.Events.Dialog', function () { setupTestBrowserHooks(); @@ -46,7 +46,7 @@ describe('Page.Events.Dialog', function () { expect(dialog.message()).toBe('yo'); }); - itFailsFirefox('should allow accepting prompts', async () => { + it('should allow accepting prompts', async () => { const {page} = getTestState(); const onDialog = sinon.stub().callsFake(dialog => { diff --git a/test/src/drag-and-drop.spec.ts b/test/src/drag-and-drop.spec.ts index 0b0f93b5c91e1..93ae07856bf2d 100644 --- a/test/src/drag-and-drop.spec.ts +++ b/test/src/drag-and-drop.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestPageAndContextHooks, setupTestBrowserHooks, - describeChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Input.drag', function () { +describe('Input.drag', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('should throw an exception if not enabled before usage', async () => { diff --git a/test/src/elementhandle.spec.ts b/test/src/elementhandle.spec.ts index b066c8c8dbe95..0d64cd48bfe39 100644 --- a/test/src/elementhandle.spec.ts +++ b/test/src/elementhandle.spec.ts @@ -18,12 +18,11 @@ import expect from 'expect'; import sinon from 'sinon'; import {ElementHandle} from '../../lib/cjs/puppeteer/common/ElementHandle.js'; import { - describeFailsFirefox, getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; import utils from './utils.js'; @@ -31,7 +30,7 @@ describe('ElementHandle specs', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('ElementHandle.boundingBox', function () { + describe('ElementHandle.boundingBox', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -96,7 +95,7 @@ describe('ElementHandle specs', function () { }); }); - describeFailsFirefox('ElementHandle.boxModel', function () { + describe('ElementHandle.boxModel', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -162,7 +161,7 @@ describe('ElementHandle specs', function () { }); describe('ElementHandle.contentFrame', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -175,7 +174,7 @@ describe('ElementHandle specs', function () { describe('ElementHandle.click', function () { // See https://github.com/puppeteer/puppeteer/issues/7175 - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/button.html'); diff --git a/test/src/emulation.spec.ts b/test/src/emulation.spec.ts index 30f6268e2e056..c275e12d24a3a 100644 --- a/test/src/emulation.spec.ts +++ b/test/src/emulation.spec.ts @@ -20,9 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Emulation', () => { setupTestBrowserHooks(); @@ -132,7 +131,7 @@ describe('Emulation', () => { }) ).toBe(true); }); - itFailsFirefox('should support landscape emulation', async () => { + it('should support landscape emulation', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/mobile.html'); @@ -173,7 +172,7 @@ describe('Emulation', () => { }) ).toContain('iPhone'); }); - itFailsFirefox('should support clicking', async () => { + it('should support clicking', async () => { const {page, server} = getTestState(); await page.emulate(iPhone); @@ -192,7 +191,7 @@ describe('Emulation', () => { }); describe('Page.emulateMediaType', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); expect( @@ -240,7 +239,7 @@ describe('Emulation', () => { }); describe('Page.emulateMediaFeatures', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); await page.emulateMediaFeatures([ @@ -370,7 +369,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateTimezone', function () { + describe('Page.emulateTimezone', function () { it('should work', async () => { const {page} = getTestState(); @@ -425,7 +424,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateVisionDeficiency', function () { + describe('Page.emulateVisionDeficiency', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -489,7 +488,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateNetworkConditions', function () { + describe('Page.emulateNetworkConditions', function () { it('should change navigator.connection.effectiveType', async () => { const {page, puppeteer} = getTestState(); @@ -511,7 +510,7 @@ describe('Emulation', () => { }); }); - describeFailsFirefox('Page.emulateCPUThrottling', function () { + describe('Page.emulateCPUThrottling', function () { it('should change the CPU throttling rate successfully', async () => { const {page} = getTestState(); diff --git a/test/src/evaluation.spec.ts b/test/src/evaluation.spec.ts index 307bf6058fcc3..b68cfdc1f2c76 100644 --- a/test/src/evaluation.spec.ts +++ b/test/src/evaluation.spec.ts @@ -20,9 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const bigint = typeof BigInt !== 'undefined'; @@ -115,18 +114,15 @@ describe('Evaluation specs', function () { await page.goto(server.PREFIX + '/global-var.html'); expect(await page.evaluate('globalVar')).toBe(123); }); - itFailsFirefox( - 'should return undefined for objects with symbols', - async () => { - const {page} = getTestState(); - - expect( - await page.evaluate(() => { - return [Symbol('foo4')]; - }) - ).toBe(undefined); - } - ); + it('should return undefined for objects with symbols', async () => { + const {page} = getTestState(); + + expect( + await page.evaluate(() => { + return [Symbol('foo4')]; + }) + ).toBe(undefined); + }); it('should work with function shorthands', async () => { const {page} = getTestState(); @@ -155,7 +151,7 @@ describe('Evaluation specs', function () { ); expect(result).toBe(42); }); - itFailsFirefox('should throw when evaluation triggers reload', async () => { + it('should throw when evaluation triggers reload', async () => { const {page} = getTestState(); let error!: Error; @@ -189,7 +185,7 @@ describe('Evaluation specs', function () { await page.goto(server.EMPTY_PAGE); expect(await frameEvaluation).toBe(42); }); - itFailsFirefox('should work from-inside an exposed function', async () => { + it('should work from-inside an exposed function', async () => { const {page} = getTestState(); // Setup inpage callback, which calls Page.evaluate @@ -324,19 +320,16 @@ describe('Evaluation specs', function () { }) ).toEqual({}); }); - itFailsFirefox( - 'should return undefined for non-serializable objects', - async () => { - const {page} = getTestState(); + it('should return undefined for non-serializable objects', async () => { + const {page} = getTestState(); - expect( - await page.evaluate(() => { - return window; - }) - ).toBe(undefined); - } - ); - itFailsFirefox('should fail for circular object', async () => { + expect( + await page.evaluate(() => { + return window; + }) + ).toBe(undefined); + }); + it('should fail for circular object', async () => { const {page} = getTestState(); const result = await page.evaluate(() => { @@ -347,7 +340,7 @@ describe('Evaluation specs', function () { }); expect(result).toBe(undefined); }); - itFailsFirefox('should be able to throw a tricky error', async () => { + it('should be able to throw a tricky error', async () => { const {page} = getTestState(); const windowHandle = await page.evaluateHandle(() => { @@ -410,28 +403,25 @@ describe('Evaluation specs', function () { }); expect(error.message).toContain('JSHandle is disposed'); }); - itFailsFirefox( - 'should throw if elementHandles are from other frames', - async () => { - const {page, server} = getTestState(); - - await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); - const bodyHandle = await page.frames()[1]!.$('body'); - let error!: Error; - await page - .evaluate(body => { - return body?.innerHTML; - }, bodyHandle) - .catch(error_ => { - return (error = error_); - }); - expect(error).toBeTruthy(); - expect(error.message).toContain( - 'JSHandles can be evaluated only in the context they were created' - ); - } - ); - itFailsFirefox('should simulate a user gesture', async () => { + it('should throw if elementHandles are from other frames', async () => { + const {page, server} = getTestState(); + + await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); + const bodyHandle = await page.frames()[1]!.$('body'); + let error!: Error; + await page + .evaluate(body => { + return body?.innerHTML; + }, bodyHandle) + .catch(error_ => { + return (error = error_); + }); + expect(error).toBeTruthy(); + expect(error.message).toContain( + 'JSHandles can be evaluated only in the context they were created' + ); + }); + it('should simulate a user gesture', async () => { const {page} = getTestState(); const result = await page.evaluate(() => { @@ -441,7 +431,7 @@ describe('Evaluation specs', function () { }); expect(result).toBe(true); }); - itFailsFirefox('should throw a nice error after a navigation', async () => { + it('should throw a nice error after a navigation', async () => { const {page} = getTestState(); const executionContext = await page.mainFrame().executionContext(); @@ -461,19 +451,16 @@ describe('Evaluation specs', function () { }); expect((error as Error).message).toContain('navigation'); }); - itFailsFirefox( - 'should not throw an error when evaluation does a navigation', - async () => { - const {page, server} = getTestState(); + it('should not throw an error when evaluation does a navigation', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/one-style.html'); - const result = await page.evaluate(() => { - (window as any).location = '/empty.html'; - return [42]; - }); - expect(result).toEqual([42]); - } - ); + await page.goto(server.PREFIX + '/one-style.html'); + const result = await page.evaluate(() => { + (window as any).location = '/empty.html'; + return [42]; + }); + expect(result).toEqual([42]); + }); it('should transfer 100Mb of data from page to node.js', async function () { const {page} = getTestState(); @@ -499,7 +486,7 @@ describe('Evaluation specs', function () { }); }); - describeFailsFirefox('Page.evaluateOnNewDocument', function () { + describe('Page.evaluateOnNewDocument', function () { it('should evaluate before anything else on the page', async () => { const {page, server} = getTestState(); diff --git a/test/src/fixtures.spec.ts b/test/src/fixtures.spec.ts index e20054dc6dcdc..4d32f73285561 100644 --- a/test/src/fixtures.spec.ts +++ b/test/src/fixtures.spec.ts @@ -17,12 +17,13 @@ /* eslint-disable @typescript-eslint/no-var-requires */ import expect from 'expect'; -import {getTestState, itHeadlessOnly} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import path from 'path'; +import {it} from './mocha-utils.js'; describe('Fixtures', function () { - itHeadlessOnly('dumpio option should work with pipe option', async () => { + it('dumpio option should work with pipe option', async () => { const {defaultBrowserOptions, puppeteerPath, headless} = getTestState(); if (headless === 'chrome') { // This test only works in the old headless mode. diff --git a/test/src/frame.spec.ts b/test/src/frame.spec.ts index b07b72769ab85..ac010d570f553 100644 --- a/test/src/frame.spec.ts +++ b/test/src/frame.spec.ts @@ -19,18 +19,18 @@ import {CDPSession} from '../../lib/cjs/puppeteer/common/Connection.js'; import {Frame} from '../../lib/cjs/puppeteer/common/Frame.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils, {dumpFrames} from './utils.js'; +import {it} from './mocha-utils.js'; describe('Frame specs', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); describe('Frame.executionContext', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -80,7 +80,7 @@ describe('Frame specs', function () { }); describe('Frame.evaluate', function () { - itFailsFirefox('should throw for detached frames', async () => { + it('should throw for detached frames', async () => { const {page, server} = getTestState(); const frame1 = (await utils.attachFrame( @@ -126,7 +126,7 @@ describe('Frame specs', function () { }); describe('Frame Management', function () { - itFailsFirefox('should handle nested frames', async () => { + it('should handle nested frames', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/frames/nested-frames.html'); @@ -138,40 +138,37 @@ describe('Frame specs', function () { ' http://localhost:/frames/frame.html (aframe)', ]); }); - itFailsFirefox( - 'should send events when frames are manipulated dynamically', - async () => { - const {page, server} = getTestState(); - - await page.goto(server.EMPTY_PAGE); - // validate frameattached events - const attachedFrames: Frame[] = []; - page.on('frameattached', frame => { - return attachedFrames.push(frame); - }); - await utils.attachFrame(page, 'frame1', './assets/frame.html'); - expect(attachedFrames.length).toBe(1); - expect(attachedFrames[0]!.url()).toContain('/assets/frame.html'); - - // validate framenavigated events - const navigatedFrames: Frame[] = []; - page.on('framenavigated', frame => { - return navigatedFrames.push(frame); - }); - await utils.navigateFrame(page, 'frame1', './empty.html'); - expect(navigatedFrames.length).toBe(1); - expect(navigatedFrames[0]!.url()).toBe(server.EMPTY_PAGE); - - // validate framedetached events - const detachedFrames: Frame[] = []; - page.on('framedetached', frame => { - return detachedFrames.push(frame); - }); - await utils.detachFrame(page, 'frame1'); - expect(detachedFrames.length).toBe(1); - expect(detachedFrames[0]!.isDetached()).toBe(true); - } - ); + it('should send events when frames are manipulated dynamically', async () => { + const {page, server} = getTestState(); + + await page.goto(server.EMPTY_PAGE); + // validate frameattached events + const attachedFrames: Frame[] = []; + page.on('frameattached', frame => { + return attachedFrames.push(frame); + }); + await utils.attachFrame(page, 'frame1', './assets/frame.html'); + expect(attachedFrames.length).toBe(1); + expect(attachedFrames[0]!.url()).toContain('/assets/frame.html'); + + // validate framenavigated events + const navigatedFrames: Frame[] = []; + page.on('framenavigated', frame => { + return navigatedFrames.push(frame); + }); + await utils.navigateFrame(page, 'frame1', './empty.html'); + expect(navigatedFrames.length).toBe(1); + expect(navigatedFrames[0]!.url()).toBe(server.EMPTY_PAGE); + + // validate framedetached events + const detachedFrames: Frame[] = []; + page.on('framedetached', frame => { + return detachedFrames.push(frame); + }); + await utils.detachFrame(page, 'frame1'); + expect(detachedFrames.length).toBe(1); + expect(detachedFrames[0]!.isDetached()).toBe(true); + }); it('should send "framenavigated" when navigating on anchor URLs', async () => { const {page, server} = getTestState(); @@ -259,7 +256,7 @@ describe('Frame specs', function () { expect(detachedFrames.length).toBe(4); expect(navigatedFrames.length).toBe(1); }); - itFailsFirefox('should report frame from-inside shadow DOM', async () => { + it('should report frame from-inside shadow DOM', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/shadow.html'); @@ -274,7 +271,7 @@ describe('Frame specs', function () { expect(page.frames().length).toBe(2); expect(page.frames()[1]!.url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox('should report frame.name()', async () => { + it('should report frame.name()', async () => { const {page, server} = getTestState(); await utils.attachFrame(page, 'theFrameId', server.EMPTY_PAGE); @@ -291,7 +288,7 @@ describe('Frame specs', function () { expect(page.frames()[1]!.name()).toBe('theFrameId'); expect(page.frames()[2]!.name()).toBe('theFrameName'); }); - itFailsFirefox('should report frame.parent()', async () => { + it('should report frame.parent()', async () => { const {page, server} = getTestState(); await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -300,31 +297,24 @@ describe('Frame specs', function () { expect(page.frames()[1]!.parentFrame()).toBe(page.mainFrame()); expect(page.frames()[2]!.parentFrame()).toBe(page.mainFrame()); }); - itFailsFirefox( - 'should report different frame instance when frame re-attaches', - async () => { - const {page, server} = getTestState(); - - const frame1 = await utils.attachFrame( - page, - 'frame1', - server.EMPTY_PAGE - ); - await page.evaluate(() => { - (globalThis as any).frame = document.querySelector('#frame1'); - (globalThis as any).frame.remove(); - }); - expect(frame1!.isDetached()).toBe(true); - const [frame2] = await Promise.all([ - utils.waitEvent(page, 'frameattached'), - page.evaluate(() => { - return document.body.appendChild((globalThis as any).frame); - }), - ]); - expect(frame2.isDetached()).toBe(false); - expect(frame1).not.toBe(frame2); - } - ); + it('should report different frame instance when frame re-attaches', async () => { + const {page, server} = getTestState(); + + const frame1 = await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); + await page.evaluate(() => { + (globalThis as any).frame = document.querySelector('#frame1'); + (globalThis as any).frame.remove(); + }); + expect(frame1!.isDetached()).toBe(true); + const [frame2] = await Promise.all([ + utils.waitEvent(page, 'frameattached'), + page.evaluate(() => { + return document.body.appendChild((globalThis as any).frame); + }), + ]); + expect(frame2.isDetached()).toBe(false); + expect(frame1).not.toBe(frame2); + }); it('should support url fragment', async () => { const {page, server} = getTestState(); @@ -335,7 +325,7 @@ describe('Frame specs', function () { server.PREFIX + '/frames/frame.html?param=value#fragment' ); }); - itFailsFirefox('should support lazy frames', async () => { + it('should support lazy frames', async () => { const {page, server} = getTestState(); await page.setViewport({width: 1000, height: 1000}); diff --git a/test/src/headful.spec.ts b/test/src/headful.spec.ts index 95abfaa2c6e4b..6ba729ab3f005 100644 --- a/test/src/headful.spec.ts +++ b/test/src/headful.spec.ts @@ -24,11 +24,8 @@ import { PuppeteerLaunchOptions, PuppeteerNode, } from '../../lib/cjs/puppeteer/node/Puppeteer.js'; -import { - describeChromeOnly, - getTestState, - itFailsWindows, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const rmAsync = promisify(rimraf); const mkdtempAsync = promisify(fs.mkdtemp); @@ -44,7 +41,7 @@ const serviceWorkerExtensionPath = path.join( 'extension' ); -describeChromeOnly('headful tests', function () { +describe('headful tests', function () { /* These tests fire up an actual browser so let's * allow a higher timeout */ @@ -214,41 +211,38 @@ describeChromeOnly('headful tests', function () { expect(pages).toEqual(['about:blank']); await browser.close(); }); - itFailsWindows( - 'headless should be able to read cookies written by headful', - async () => { - /* Needs investigation into why but this fails consistently on Windows CI. */ - const {server, puppeteer} = getTestState(); - - const userDataDir = await mkdtempAsync(TMP_FOLDER); - // Write a cookie in headful chrome - const headfulBrowser = await launchBrowser( - puppeteer, - Object.assign({userDataDir}, headfulOptions) - ); - const headfulPage = await headfulBrowser.newPage(); - await headfulPage.goto(server.EMPTY_PAGE); - await headfulPage.evaluate(() => { - return (document.cookie = - 'foo=true; expires=Fri, 31 Dec 9999 23:59:59 GMT'); - }); - await headfulBrowser.close(); - // Read the cookie from headless chrome - const headlessBrowser = await launchBrowser( - puppeteer, - Object.assign({userDataDir}, headlessOptions) - ); - const headlessPage = await headlessBrowser.newPage(); - await headlessPage.goto(server.EMPTY_PAGE); - const cookie = await headlessPage.evaluate(() => { - return document.cookie; - }); - await headlessBrowser.close(); - // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 - await rmAsync(userDataDir).catch(() => {}); - expect(cookie).toBe('foo=true'); - } - ); + it('headless should be able to read cookies written by headful', async () => { + /* Needs investigation into why but this fails consistently on Windows CI. */ + const {server, puppeteer} = getTestState(); + + const userDataDir = await mkdtempAsync(TMP_FOLDER); + // Write a cookie in headful chrome + const headfulBrowser = await launchBrowser( + puppeteer, + Object.assign({userDataDir}, headfulOptions) + ); + const headfulPage = await headfulBrowser.newPage(); + await headfulPage.goto(server.EMPTY_PAGE); + await headfulPage.evaluate(() => { + return (document.cookie = + 'foo=true; expires=Fri, 31 Dec 9999 23:59:59 GMT'); + }); + await headfulBrowser.close(); + // Read the cookie from headless chrome + const headlessBrowser = await launchBrowser( + puppeteer, + Object.assign({userDataDir}, headlessOptions) + ); + const headlessPage = await headlessBrowser.newPage(); + await headlessPage.goto(server.EMPTY_PAGE); + const cookie = await headlessPage.evaluate(() => { + return document.cookie; + }); + await headlessBrowser.close(); + // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 + await rmAsync(userDataDir).catch(() => {}); + expect(cookie).toBe('foo=true'); + }); // TODO: Support OOOPIF. @see https://github.com/puppeteer/puppeteer/issues/2548 xit('OOPIF: should report google.com frame', async () => { const {server, puppeteer} = getTestState(); diff --git a/test/src/idle_override.spec.ts b/test/src/idle_override.spec.ts index 89d8ecf9683ff..1dae41d0c91f7 100644 --- a/test/src/idle_override.spec.ts +++ b/test/src/idle_override.spec.ts @@ -20,10 +20,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Emulate idle state', () => { +describe('Emulate idle state', () => { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/ignorehttpserrors.spec.ts b/test/src/ignorehttpserrors.spec.ts index 503863808c7ab..f4f5e6da8a34b 100644 --- a/test/src/ignorehttpserrors.spec.ts +++ b/test/src/ignorehttpserrors.spec.ts @@ -22,11 +22,8 @@ import { } from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; -import { - getTestState, - describeFailsFirefox, - itFailsFirefox, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('ignoreHTTPSErrors', function () { /* Note that this test creates its own browser rather than use @@ -59,7 +56,7 @@ describe('ignoreHTTPSErrors', function () { await context.close(); }); - describeFailsFirefox('Response.securityDetails', function () { + describe('Response.securityDetails', function () { it('should work', async () => { const {httpsServer} = getTestState(); @@ -119,7 +116,7 @@ describe('ignoreHTTPSErrors', function () { expect(error).toBeUndefined(); expect(response.ok()).toBe(true); }); - itFailsFirefox('should work with request interception', async () => { + it('should work with request interception', async () => { const {httpsServer} = getTestState(); await page.setRequestInterception(true); @@ -129,7 +126,7 @@ describe('ignoreHTTPSErrors', function () { const response = (await page.goto(httpsServer.EMPTY_PAGE))!; expect(response.status()).toBe(200); }); - itFailsFirefox('should work with mixed content', async () => { + it('should work with mixed content', async () => { const {server, httpsServer} = getTestState(); httpsServer.setRoute('/mixedcontent.html', (_req, res) => { diff --git a/test/src/input.spec.ts b/test/src/input.spec.ts index 831c362e40fd4..c06f4dd0b4669 100644 --- a/test/src/input.spec.ts +++ b/test/src/input.spec.ts @@ -20,8 +20,8 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; const FILE_TO_UPLOAD = path.join(__dirname, '/../assets/file-to-upload.txt'); @@ -29,7 +29,7 @@ describe('input tests', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('input', function () { + describe('input', function () { it('should upload the file', async () => { const {page, server} = getTestState(); @@ -76,7 +76,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('Page.waitForFileChooser', function () { + describe('Page.waitForFileChooser', function () { it('should work when file input is attached to DOM', async () => { const {page} = getTestState(); @@ -159,7 +159,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.accept', function () { + describe('FileChooser.accept', function () { it('should accept single file', async () => { const {page} = getTestState(); @@ -325,7 +325,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.cancel', function () { + describe('FileChooser.cancel', function () { it('should cancel dialog', async () => { const {page} = getTestState(); @@ -373,7 +373,7 @@ describe('input tests', function () { }); }); - describeFailsFirefox('FileChooser.isMultiple', () => { + describe('FileChooser.isMultiple', () => { it('should work for single file pick', async () => { const {page} = getTestState(); diff --git a/test/src/jshandle.spec.ts b/test/src/jshandle.spec.ts index 7147fffb99fba..a97925f778161 100644 --- a/test/src/jshandle.spec.ts +++ b/test/src/jshandle.spec.ts @@ -17,11 +17,11 @@ import expect from 'expect'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, shortWaitForArrayToHaveAtLeastNElements, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('JSHandle', function () { setupTestBrowserHooks(); @@ -159,7 +159,7 @@ describe('JSHandle', function () { expect(await bHandle.jsonValue()).toEqual(undefined); }); - itFailsFirefox('should not work with dates', async () => { + it('should not work with dates', async () => { const {page} = getTestState(); const dateHandle = await page.evaluateHandle(() => { @@ -409,7 +409,7 @@ describe('JSHandle', function () { }); describe('JSHandle.click', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); const clicks: Array<[x: number, y: number]> = []; diff --git a/test/src/keyboard.spec.ts b/test/src/keyboard.spec.ts index 43ce79b207ead..812702963400c 100644 --- a/test/src/keyboard.spec.ts +++ b/test/src/keyboard.spec.ts @@ -21,9 +21,9 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; import {KeyInput} from '../../lib/cjs/puppeteer/common/USKeyboardLayout.js'; +import {it} from './mocha-utils.js'; describe('Keyboard', function () { setupTestBrowserHooks(); @@ -45,7 +45,7 @@ describe('Keyboard', function () { }) ).toBe(text); }); - itFailsFirefox('should press the metaKey', async () => { + it('should press the metaKey', async () => { const {page, isFirefox} = getTestState(); await page.evaluate(() => { @@ -120,22 +120,19 @@ describe('Keyboard', function () { }) ).toBe('a'); }); - itFailsFirefox( - 'ElementHandle.press should support |text| option', - async () => { - const {page, server} = getTestState(); - - await page.goto(server.PREFIX + '/input/textarea.html'); - const textarea = (await page.$('textarea'))!; - await textarea.press('a', {text: 'ё'}); - expect( - await page.evaluate(() => { - return document.querySelector('textarea')!.value; - }) - ).toBe('ё'); - } - ); - itFailsFirefox('should send a character with sendCharacter', async () => { + it('ElementHandle.press should support |text| option', async () => { + const {page, server} = getTestState(); + + await page.goto(server.PREFIX + '/input/textarea.html'); + const textarea = (await page.$('textarea'))!; + await textarea.press('a', {text: 'ё'}); + expect( + await page.evaluate(() => { + return document.querySelector('textarea')!.value; + }) + ).toBe('ё'); + }); + it('should send a character with sendCharacter', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -162,7 +159,7 @@ describe('Keyboard', function () { }) ).toBe('嗨a'); }); - itFailsFirefox('should report shiftKey', async () => { + it('should report shiftKey', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/keyboard.html'); @@ -353,7 +350,7 @@ describe('Keyboard', function () { }) ).toBe('He Wrd!'); }); - itFailsFirefox('should specify repeat property', async () => { + it('should specify repeat property', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -401,7 +398,7 @@ describe('Keyboard', function () { }) ).toBe(false); }); - itFailsFirefox('should type all kinds of characters', async () => { + it('should type all kinds of characters', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -410,7 +407,7 @@ describe('Keyboard', function () { await page.keyboard.type(text); expect(await page.evaluate('result')).toBe(text); }); - itFailsFirefox('should specify location', async () => { + it('should specify location', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -460,7 +457,7 @@ describe('Keyboard', function () { }); expect(error && error.message).toBe('Unknown key: "😊"'); }); - itFailsFirefox('should type emoji', async () => { + it('should type emoji', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/textarea.html'); @@ -471,7 +468,7 @@ describe('Keyboard', function () { }) ).toBe('👹 Tokyo street Japan 🇯🇵'); }); - itFailsFirefox('should type emoji into an iframe', async () => { + it('should type emoji into an iframe', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -489,7 +486,7 @@ describe('Keyboard', function () { }) ).toBe('👹 Tokyo street Japan 🇯🇵'); }); - itFailsFirefox('should press the meta key', async () => { + it('should press the meta key', async () => { const {page, isFirefox} = getTestState(); await page.evaluate(() => { diff --git a/test/src/launcher.spec.ts b/test/src/launcher.spec.ts index a814967aa41c9..01e01e5cc9bd0 100644 --- a/test/src/launcher.spec.ts +++ b/test/src/launcher.spec.ts @@ -24,14 +24,9 @@ import {TLSSocket} from 'tls'; import {promisify} from 'util'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {Product} from '../../lib/cjs/puppeteer/common/Product.js'; -import { - getTestState, - itChromeOnly, - itFailsFirefox, - itFirefoxOnly, - itOnlyRegularInstall, -} from './mocha-utils.js'; +import {getTestState, itOnlyRegularInstall} from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; const mkdtempAsync = promisify(fs.mkdtemp); const readFileAsync = promisify(fs.readFile); @@ -251,7 +246,7 @@ describe('Launcher specs', function () { // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 await rmAsync(userDataDir).catch(() => {}); }); - itChromeOnly('tmp profile should be cleaned up', async () => { + it('tmp profile should be cleaned up', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); // Set a custom test tmp dir so that we can validate that @@ -280,7 +275,7 @@ describe('Launcher specs', function () { // Restore env var process.env['PUPPETEER_TMP_DIR'] = ''; }); - itFirefoxOnly('userDataDir option restores preferences', async () => { + it('userDataDir option restores preferences', async () => { const {defaultBrowserOptions, puppeteer} = getTestState(); const userDataDir = await mkdtempAsync(TMP_FOLDER); @@ -326,7 +321,7 @@ describe('Launcher specs', function () { // This might throw. See https://github.com/puppeteer/puppeteer/issues/2778 await rmAsync(userDataDir).catch(() => {}); }); - itChromeOnly('userDataDir argument with non-existent dir', async () => { + it('userDataDir argument with non-existent dir', async () => { const {isChrome, puppeteer, defaultBrowserOptions} = getTestState(); const userDataDir = await mkdtempAsync(TMP_FOLDER); @@ -460,49 +455,43 @@ describe('Launcher specs', function () { await page.close(); await browser.close(); }); - itChromeOnly( - 'should filter out ignored default arguments in Chrome', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); - // Make sure we launch with `--enable-automation` by default. - const defaultArgs = puppeteer.defaultArgs(); - const browser = await puppeteer.launch( - Object.assign({}, defaultBrowserOptions, { - // Ignore first and third default argument. - ignoreDefaultArgs: [defaultArgs[0]!, defaultArgs[2]], - }) - ); - const spawnargs = browser.process()!.spawnargs; - if (!spawnargs) { - throw new Error('spawnargs not present'); - } - expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); - expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); - expect(spawnargs.indexOf(defaultArgs[2]!)).toBe(-1); - await browser.close(); + it('should filter out ignored default arguments in Chrome', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); + // Make sure we launch with `--enable-automation` by default. + const defaultArgs = puppeteer.defaultArgs(); + const browser = await puppeteer.launch( + Object.assign({}, defaultBrowserOptions, { + // Ignore first and third default argument. + ignoreDefaultArgs: [defaultArgs[0]!, defaultArgs[2]], + }) + ); + const spawnargs = browser.process()!.spawnargs; + if (!spawnargs) { + throw new Error('spawnargs not present'); } - ); - itFirefoxOnly( - 'should filter out ignored default argument in Firefox', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); - - const defaultArgs = puppeteer.defaultArgs(); - const browser = await puppeteer.launch( - Object.assign({}, defaultBrowserOptions, { - // Only the first argument is fixed, others are optional. - ignoreDefaultArgs: [defaultArgs[0]!], - }) - ); - const spawnargs = browser.process()!.spawnargs; - if (!spawnargs) { - throw new Error('spawnargs not present'); - } - expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); - expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); - await browser.close(); + expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); + expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); + expect(spawnargs.indexOf(defaultArgs[2]!)).toBe(-1); + await browser.close(); + }); + it('should filter out ignored default argument in Firefox', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); + + const defaultArgs = puppeteer.defaultArgs(); + const browser = await puppeteer.launch( + Object.assign({}, defaultBrowserOptions, { + // Only the first argument is fixed, others are optional. + ignoreDefaultArgs: [defaultArgs[0]!], + }) + ); + const spawnargs = browser.process()!.spawnargs; + if (!spawnargs) { + throw new Error('spawnargs not present'); } - ); + expect(spawnargs.indexOf(defaultArgs[0]!)).toBe(-1); + expect(spawnargs.indexOf(defaultArgs[1]!)).not.toBe(-1); + await browser.close(); + }); it('should have default URL when launching browser', async function () { const {defaultBrowserOptions, puppeteer} = getTestState(); const browser = await puppeteer.launch(defaultBrowserOptions); @@ -512,24 +501,21 @@ describe('Launcher specs', function () { expect(pages).toEqual(['about:blank']); await browser.close(); }); - itFailsFirefox( - 'should have custom URL when launching browser', - async () => { - const {server, puppeteer, defaultBrowserOptions} = getTestState(); - - const options = Object.assign({}, defaultBrowserOptions); - options.args = [server.EMPTY_PAGE].concat(options.args || []); - const browser = await puppeteer.launch(options); - const pages = await browser.pages(); - expect(pages.length).toBe(1); - const page = pages[0]!; - if (page.url() !== server.EMPTY_PAGE) { - await page.waitForNavigation(); - } - expect(page.url()).toBe(server.EMPTY_PAGE); - await browser.close(); + it('should have custom URL when launching browser', async () => { + const {server, puppeteer, defaultBrowserOptions} = getTestState(); + + const options = Object.assign({}, defaultBrowserOptions); + options.args = [server.EMPTY_PAGE].concat(options.args || []); + const browser = await puppeteer.launch(options); + const pages = await browser.pages(); + expect(pages.length).toBe(1); + const page = pages[0]!; + if (page.url() !== server.EMPTY_PAGE) { + await page.waitForNavigation(); } - ); + expect(page.url()).toBe(server.EMPTY_PAGE); + await browser.close(); + }); it('should pass the timeout parameter to browser.waitForTarget', async () => { const {puppeteer, defaultBrowserOptions} = getTestState(); const options = Object.assign({}, defaultBrowserOptions, { @@ -615,24 +601,21 @@ describe('Launcher specs', function () { }); expect(error.message).toContain('either pipe or debugging port'); }); - itChromeOnly( - 'should launch Chrome properly with --no-startup-window and waitForInitialPage=false', - async () => { - const {defaultBrowserOptions, puppeteer} = getTestState(); - const options = { - waitForInitialPage: false, - // This is needed to prevent Puppeteer from adding an initial blank page. - // See also https://github.com/puppeteer/puppeteer/blob/ad6b736039436fcc5c0a262e5b575aa041427be3/src/node/Launcher.ts#L200 - ignoreDefaultArgs: true, - ...defaultBrowserOptions, - args: ['--no-startup-window'], - }; - const browser = await puppeteer.launch(options); - const pages = await browser.pages(); - expect(pages.length).toBe(0); - await browser.close(); - } - ); + it('should launch Chrome properly with --no-startup-window and waitForInitialPage=false', async () => { + const {defaultBrowserOptions, puppeteer} = getTestState(); + const options = { + waitForInitialPage: false, + // This is needed to prevent Puppeteer from adding an initial blank page. + // See also https://github.com/puppeteer/puppeteer/blob/ad6b736039436fcc5c0a262e5b575aa041427be3/src/node/Launcher.ts#L200 + ignoreDefaultArgs: true, + ...defaultBrowserOptions, + args: ['--no-startup-window'], + }; + const browser = await puppeteer.launch(options); + const pages = await browser.pages(); + expect(pages.length).toBe(0); + await browser.close(); + }); }); describe('Puppeteer.launch', function () { @@ -775,7 +758,7 @@ describe('Launcher specs', function () { }); // @see https://github.com/puppeteer/puppeteer/issues/4197 - itFailsFirefox('should support targetFilter option', async () => { + it('should support targetFilter option', async () => { const {server, puppeteer, defaultBrowserOptions} = getTestState(); const originalBrowser = await puppeteer.launch(defaultBrowserOptions); @@ -809,68 +792,62 @@ describe('Launcher specs', function () { .sort() ).toEqual(['about:blank', server.EMPTY_PAGE]); }); - itFailsFirefox( - 'should be able to reconnect to a disconnected browser', - async () => { - const {server, puppeteer, defaultBrowserOptions} = getTestState(); - - const originalBrowser = await puppeteer.launch(defaultBrowserOptions); - const browserWSEndpoint = originalBrowser.wsEndpoint(); - const page = await originalBrowser.newPage(); - await page.goto(server.PREFIX + '/frames/nested-frames.html'); - originalBrowser.disconnect(); - - const browser = await puppeteer.connect({browserWSEndpoint}); - const pages = await browser.pages(); - const restoredPage = pages.find(page => { - return page.url() === server.PREFIX + '/frames/nested-frames.html'; - })!; - expect(utils.dumpFrames(restoredPage.mainFrame())).toEqual([ - 'http://localhost:/frames/nested-frames.html', - ' http://localhost:/frames/two-frames.html (2frames)', - ' http://localhost:/frames/frame.html (uno)', - ' http://localhost:/frames/frame.html (dos)', - ' http://localhost:/frames/frame.html (aframe)', - ]); - expect( - await restoredPage.evaluate(() => { - return 7 * 8; - }) - ).toBe(56); - await browser.close(); - } - ); + it('should be able to reconnect to a disconnected browser', async () => { + const {server, puppeteer, defaultBrowserOptions} = getTestState(); + + const originalBrowser = await puppeteer.launch(defaultBrowserOptions); + const browserWSEndpoint = originalBrowser.wsEndpoint(); + const page = await originalBrowser.newPage(); + await page.goto(server.PREFIX + '/frames/nested-frames.html'); + originalBrowser.disconnect(); + + const browser = await puppeteer.connect({browserWSEndpoint}); + const pages = await browser.pages(); + const restoredPage = pages.find(page => { + return page.url() === server.PREFIX + '/frames/nested-frames.html'; + })!; + expect(utils.dumpFrames(restoredPage.mainFrame())).toEqual([ + 'http://localhost:/frames/nested-frames.html', + ' http://localhost:/frames/two-frames.html (2frames)', + ' http://localhost:/frames/frame.html (uno)', + ' http://localhost:/frames/frame.html (dos)', + ' http://localhost:/frames/frame.html (aframe)', + ]); + expect( + await restoredPage.evaluate(() => { + return 7 * 8; + }) + ).toBe(56); + await browser.close(); + }); // @see https://github.com/puppeteer/puppeteer/issues/4197#issuecomment-481793410 - itFailsFirefox( - 'should be able to connect to the same page simultaneously', - async () => { - const {puppeteer, defaultBrowserOptions} = getTestState(); - - const browserOne = await puppeteer.launch(defaultBrowserOptions); - const browserTwo = await puppeteer.connect({ - browserWSEndpoint: browserOne.wsEndpoint(), - }); - const [page1, page2] = await Promise.all([ - new Promise(x => { - return browserOne.once('targetcreated', target => { - return x(target.page()); - }); - }), - browserTwo.newPage(), - ]); - expect( - await page1.evaluate(() => { - return 7 * 8; - }) - ).toBe(56); - expect( - await page2.evaluate(() => { - return 7 * 6; - }) - ).toBe(42); - await browserOne.close(); - } - ); + it('should be able to connect to the same page simultaneously', async () => { + const {puppeteer, defaultBrowserOptions} = getTestState(); + + const browserOne = await puppeteer.launch(defaultBrowserOptions); + const browserTwo = await puppeteer.connect({ + browserWSEndpoint: browserOne.wsEndpoint(), + }); + const [page1, page2] = await Promise.all([ + new Promise(x => { + return browserOne.once('targetcreated', target => { + return x(target.page()); + }); + }), + browserTwo.newPage(), + ]); + expect( + await page1.evaluate(() => { + return 7 * 8; + }) + ).toBe(56); + expect( + await page2.evaluate(() => { + return 7 * 6; + }) + ).toBe(42); + await browserOne.close(); + }); it('should be able to reconnect', async () => { const {puppeteer, server, defaultBrowserOptions} = getTestState(); const browserOne = await puppeteer.launch(defaultBrowserOptions); @@ -933,7 +910,7 @@ describe('Launcher specs', function () { describe('when the product is chrome, platform is not darwin, and arch is arm64', () => { describe('and the executable exists', () => { - itChromeOnly('returns /usr/bin/chromium-browser', async () => { + it('returns /usr/bin/chromium-browser', async () => { const {puppeteer} = getTestState(); const osPlatformStub = sinon.stub(os, 'platform').returns('linux'); const osArchStub = sinon.stub(os, 'arch').returns('arm64'); @@ -972,33 +949,28 @@ describe('Launcher specs', function () { }); }); describe('and the executable does not exist', () => { - itChromeOnly( - 'does not return /usr/bin/chromium-browser', - async () => { - const {puppeteer} = getTestState(); - const osPlatformStub = sinon - .stub(os, 'platform') - .returns('linux'); - const osArchStub = sinon.stub(os, 'arch').returns('arm64'); - const fsExistsStub = sinon.stub(fs, 'existsSync'); - fsExistsStub.withArgs('/usr/bin/chromium-browser').returns(false); + it('does not return /usr/bin/chromium-browser', async () => { + const {puppeteer} = getTestState(); + const osPlatformStub = sinon.stub(os, 'platform').returns('linux'); + const osArchStub = sinon.stub(os, 'arch').returns('arm64'); + const fsExistsStub = sinon.stub(fs, 'existsSync'); + fsExistsStub.withArgs('/usr/bin/chromium-browser').returns(false); - const executablePath = puppeteer.executablePath(); + const executablePath = puppeteer.executablePath(); - expect(executablePath).not.toEqual('/usr/bin/chromium-browser'); + expect(executablePath).not.toEqual('/usr/bin/chromium-browser'); - osPlatformStub.restore(); - osArchStub.restore(); - fsExistsStub.restore(); - } - ); + osPlatformStub.restore(); + osArchStub.restore(); + fsExistsStub.restore(); + }); }); }); }); }); describe('Browser target events', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {server, puppeteer, defaultBrowserOptions} = getTestState(); const browser = await puppeteer.launch(defaultBrowserOptions); @@ -1021,51 +993,48 @@ describe('Launcher specs', function () { }); describe('Browser.Events.disconnected', function () { - itFailsFirefox( - 'should be emitted when: browser gets closed, disconnected or underlying websocket gets closed', - async () => { - const {puppeteer, defaultBrowserOptions} = getTestState(); - const originalBrowser = await puppeteer.launch(defaultBrowserOptions); - const browserWSEndpoint = originalBrowser.wsEndpoint(); - const remoteBrowser1 = await puppeteer.connect({ - browserWSEndpoint, - }); - const remoteBrowser2 = await puppeteer.connect({ - browserWSEndpoint, - }); + it('should be emitted when: browser gets closed, disconnected or underlying websocket gets closed', async () => { + const {puppeteer, defaultBrowserOptions} = getTestState(); + const originalBrowser = await puppeteer.launch(defaultBrowserOptions); + const browserWSEndpoint = originalBrowser.wsEndpoint(); + const remoteBrowser1 = await puppeteer.connect({ + browserWSEndpoint, + }); + const remoteBrowser2 = await puppeteer.connect({ + browserWSEndpoint, + }); - let disconnectedOriginal = 0; - let disconnectedRemote1 = 0; - let disconnectedRemote2 = 0; - originalBrowser.on('disconnected', () => { - return ++disconnectedOriginal; - }); - remoteBrowser1.on('disconnected', () => { - return ++disconnectedRemote1; - }); - remoteBrowser2.on('disconnected', () => { - return ++disconnectedRemote2; - }); + let disconnectedOriginal = 0; + let disconnectedRemote1 = 0; + let disconnectedRemote2 = 0; + originalBrowser.on('disconnected', () => { + return ++disconnectedOriginal; + }); + remoteBrowser1.on('disconnected', () => { + return ++disconnectedRemote1; + }); + remoteBrowser2.on('disconnected', () => { + return ++disconnectedRemote2; + }); - await Promise.all([ - utils.waitEvent(remoteBrowser2, 'disconnected'), - remoteBrowser2.disconnect(), - ]); + await Promise.all([ + utils.waitEvent(remoteBrowser2, 'disconnected'), + remoteBrowser2.disconnect(), + ]); - expect(disconnectedOriginal).toBe(0); - expect(disconnectedRemote1).toBe(0); - expect(disconnectedRemote2).toBe(1); + expect(disconnectedOriginal).toBe(0); + expect(disconnectedRemote1).toBe(0); + expect(disconnectedRemote2).toBe(1); - await Promise.all([ - utils.waitEvent(remoteBrowser1, 'disconnected'), - utils.waitEvent(originalBrowser, 'disconnected'), - originalBrowser.close(), - ]); + await Promise.all([ + utils.waitEvent(remoteBrowser1, 'disconnected'), + utils.waitEvent(originalBrowser, 'disconnected'), + originalBrowser.close(), + ]); - expect(disconnectedOriginal).toBe(1); - expect(disconnectedRemote1).toBe(1); - expect(disconnectedRemote2).toBe(1); - } - ); + expect(disconnectedOriginal).toBe(1); + expect(disconnectedRemote1).toBe(1); + expect(disconnectedRemote2).toBe(1); + }); }); }); diff --git a/test/src/mocha-utils.ts b/test/src/mocha-utils.ts index 709a67158f3b1..d21196c1e1aac 100644 --- a/test/src/mocha-utils.ts +++ b/test/src/mocha-utils.ts @@ -17,7 +17,6 @@ import Protocol from 'devtools-protocol'; import expect from 'expect'; import * as fs from 'fs'; -import * as os from 'os'; import * as path from 'path'; import rimraf from 'rimraf'; import sinon from 'sinon'; @@ -34,6 +33,8 @@ import { import puppeteer from '../../lib/cjs/puppeteer/puppeteer.js'; import {TestServer} from '../../utils/testserver/lib/index.js'; import {extendExpectWithToBeGolden} from './utils.js'; +import * as Mocha from 'mocha'; +import {getTestId} from '../../utils/mochaRunner/lib/utils.js'; const setupServer = async () => { const assetsPath = path.join(__dirname, '../assets'); @@ -63,14 +64,14 @@ export const getTestState = (): PuppeteerTestState => { }; const product = - process.env['PRODUCT'] || process.env['PUPPETEER_PRODUCT'] || 'Chromium'; + process.env['PRODUCT'] || process.env['PUPPETEER_PRODUCT'] || 'chrome'; const alternativeInstall = process.env['PUPPETEER_ALT_INSTALL'] || false; const headless = (process.env['HEADLESS'] || 'true').trim().toLowerCase(); const isHeadless = headless === 'true' || headless === 'chrome'; const isFirefox = product === 'firefox'; -const isChrome = product === 'Chromium'; +const isChrome = product === 'chrome'; let extraLaunchOptions = {}; try { @@ -125,7 +126,11 @@ declare module 'expect/build/types' { } const setupGoldenAssertions = (): void => { - const suffix = product.toLowerCase(); + let suffix = product.toLowerCase(); + if (suffix === 'chrome') { + // TODO: to avoid moving golden folders. + suffix = 'chromium'; + } const GOLDEN_DIR = path.join(__dirname, `../golden-${suffix}`); const OUTPUT_DIR = path.join(__dirname, `../output-${suffix}`); if (fs.existsSync(OUTPUT_DIR)) { @@ -152,64 +157,9 @@ interface PuppeteerTestState { } const state: Partial = {}; -export const itFailsFirefox = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isFirefox) { - return xit(description, body); - } else { - return it(description, body); - } -}; - -export const itChromeOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itHeadlessOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome && isHeadless === true) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itHeadfulOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isChrome && isHeadless === false) { - return it(description, body); - } else { - return xit(description, body); - } -}; - -export const itFirefoxOnly = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (isFirefox) { - return it(description, body); - } else { - return xit(description, body); - } -}; - export const itOnlyRegularInstall = ( description: string, - body: Mocha.Func + body: Mocha.AsyncFunc ): Mocha.Test => { if (alternativeInstall || process.env['BINARY']) { return xit(description, body); @@ -218,50 +168,10 @@ export const itOnlyRegularInstall = ( } }; -export const itFailsWindowsUntilDate = ( - date: Date, - description: string, - body: Mocha.Func -): Mocha.Test => { - if (os.platform() === 'win32' && Date.now() < date.getTime()) { - // we are within the deferred time so skip the test - return xit(description, body); - } - - return it(description, body); -}; - -export const itFailsWindows = ( - description: string, - body: Mocha.Func -): Mocha.Test => { - if (os.platform() === 'win32') { - return xit(description, body); - } - return it(description, body); -}; - -export const describeFailsFirefox = ( - description: string, - body: (this: Mocha.Suite) => void -): void | Mocha.Suite => { - if (isFirefox) { - return xdescribe(description, body); - } else { - return describe(description, body); - } -}; - -export const describeChromeOnly = ( - description: string, - body: (this: Mocha.Suite) => void -): Mocha.Suite | void => { - if (isChrome) { - return describe(description, body); - } -}; - -if (process.env['MOCHA_WORKER_ID'] === '0') { +if ( + process.env['MOCHA_WORKER_ID'] === undefined || + process.env['MOCHA_WORKER_ID'] === '0' +) { console.log( `Running unit tests with: -> product: ${product} @@ -290,7 +200,7 @@ export const setupTestBrowserHooks = (): void => { }); after(async () => { - await state.browser!.close(); + await state.browser?.close(); state.browser = undefined; }); }; @@ -302,7 +212,7 @@ export const setupTestPageAndContextHooks = (): void => { }); afterEach(async () => { - await state.context!.close(); + await state.context?.close(); state.context = undefined; state.page = undefined; }); @@ -387,3 +297,34 @@ export const shortWaitForArrayToHaveAtLeastNElements = async ( }); } }; + +type SyncFn = (this: Mocha.Context) => void; + +const skippedTests: string[] = process.env['PUPPETEER_SKIPPED_TEST_CONFIG'] + ? JSON.parse(process.env['PUPPETEER_SKIPPED_TEST_CONFIG']) + : []; + +function skipTestIfNeeded(test: Mocha.Test): void { + const testId = getTestId(test.file!, test.fullTitle()); + if ( + skippedTests.find(skippedTest => { + return testId.startsWith(skippedTest); + }) + ) { + try { + test.skip(); + } catch {} + } +} + +export function it(title: string, fn?: Mocha.AsyncFunc | SyncFn): Mocha.Test { + const test = Mocha.it.call(null, title, fn as any); + skipTestIfNeeded(test); + return test; +} + +it.only = function (title: string, fn?: Mocha.AsyncFunc | SyncFn): Mocha.Test { + const test = Mocha.it.only.call(null, title, fn as any); + skipTestIfNeeded(test); + return test; +}; diff --git a/test/src/mouse.spec.ts b/test/src/mouse.spec.ts index 078941e6b6ff2..661c2e8c2d538 100644 --- a/test/src/mouse.spec.ts +++ b/test/src/mouse.spec.ts @@ -19,9 +19,9 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, } from './mocha-utils.js'; import {KeyInput} from '../../lib/cjs/puppeteer/common/USKeyboardLayout.js'; +import {it} from './mocha-utils.js'; interface Dimensions { x: number; @@ -115,7 +115,7 @@ describe('Mouse', function () { }) ).toBe(text); }); - itFailsFirefox('should trigger hover state', async () => { + it('should trigger hover state', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/scrollable.html'); @@ -138,24 +138,21 @@ describe('Mouse', function () { }) ).toBe('button-91'); }); - itFailsFirefox( - 'should trigger hover state with removed window.Node', - async () => { - const {page, server} = getTestState(); + it('should trigger hover state with removed window.Node', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/scrollable.html'); + await page.goto(server.PREFIX + '/input/scrollable.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return delete window.Node; + }); + await page.hover('#button-6'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return delete window.Node; - }); - await page.hover('#button-6'); - expect( - await page.evaluate(() => { - return document.querySelector('button:hover')!.id; - }) - ).toBe('button-6'); - } - ); + return document.querySelector('button:hover')!.id; + }) + ).toBe('button-6'); + }); it('should set modifier keys on click', async () => { const {page, server, isFirefox} = getTestState(); @@ -202,7 +199,7 @@ describe('Mouse', function () { } } }); - itFailsFirefox('should send mouse wheel events', async () => { + it('should send mouse wheel events', async () => { const {page, server} = getTestState(); await page.goto(server.PREFIX + '/input/wheel.html'); @@ -225,7 +222,7 @@ describe('Mouse', function () { height: 230, }); }); - itFailsFirefox('should tween mouse movement', async () => { + it('should tween mouse movement', async () => { const {page} = getTestState(); await page.mouse.move(100, 100); diff --git a/test/src/navigation.spec.ts b/test/src/navigation.spec.ts index e040f23de66ce..98efda2a9a548 100644 --- a/test/src/navigation.spec.ts +++ b/test/src/navigation.spec.ts @@ -20,12 +20,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, } from './mocha-utils.js'; import os from 'os'; import {ServerResponse} from 'http'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; +import {it} from './mocha-utils.js'; describe('navigation', function () { setupTestBrowserHooks(); @@ -67,7 +66,7 @@ describe('navigation', function () { const response = (await page.goto(server.PREFIX + '/historyapi.html'))!; expect(response.status()).toBe(200); }); - itFailsFirefox('should work with subframes return 204', async () => { + it('should work with subframes return 204', async () => { const {page, server} = getTestState(); server.setRoute('/frames/frame.html', (_req, res) => { @@ -82,7 +81,7 @@ describe('navigation', function () { }); expect(error).toBeUndefined(); }); - itFailsFirefox('should fail when server returns 204', async () => { + it('should fail when server returns 204', async () => { const {page, server, isChrome} = getTestState(); server.setRoute('/empty.html', (_req, res) => { @@ -124,29 +123,23 @@ describe('navigation', function () { const response = await page.goto(server.PREFIX + '/grid.html'); expect(response!.status()).toBe(200); }); - itFailsFirefox( - 'should navigate to empty page with networkidle0', - async () => { - const {page, server} = getTestState(); + it('should navigate to empty page with networkidle0', async () => { + const {page, server} = getTestState(); - const response = await page.goto(server.EMPTY_PAGE, { - waitUntil: 'networkidle0', - }); - expect(response!.status()).toBe(200); - } - ); - itFailsFirefox( - 'should navigate to empty page with networkidle2', - async () => { - const {page, server} = getTestState(); - - const response = await page.goto(server.EMPTY_PAGE, { - waitUntil: 'networkidle2', - }); - expect(response!.status()).toBe(200); - } - ); - itFailsFirefox('should fail when navigating to bad url', async () => { + const response = await page.goto(server.EMPTY_PAGE, { + waitUntil: 'networkidle0', + }); + expect(response!.status()).toBe(200); + }); + it('should navigate to empty page with networkidle2', async () => { + const {page, server} = getTestState(); + + const response = await page.goto(server.EMPTY_PAGE, { + waitUntil: 'networkidle2', + }); + expect(response!.status()).toBe(200); + }); + it('should fail when navigating to bad url', async () => { const {page, isChrome} = getTestState(); let error!: Error; @@ -175,7 +168,7 @@ describe('navigation', function () { : 'net::ERR_CERT_AUTHORITY_INVALID'; } - itFailsFirefox('should fail when navigating to bad SSL', async () => { + it('should fail when navigating to bad SSL', async () => { const {page, httpsServer, isChrome} = getTestState(); // Make sure that network events do not emit 'undefined'. @@ -311,7 +304,7 @@ describe('navigation', function () { const response = (await page.goto(server.EMPTY_PAGE))!; expect(response.ok()).toBe(true); }); - itFailsFirefox('should work when navigating to data url', async () => { + it('should work when navigating to data url', async () => { const {page} = getTestState(); const response = (await page.goto('data:text/html,hello'))!; @@ -334,85 +327,79 @@ describe('navigation', function () { expect(response.ok()).toBe(true); expect(response.url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox( - 'should wait for network idle to succeed navigation', - async () => { - const {page, server} = getTestState(); - - let responses: ServerResponse[] = []; - // Hold on to a bunch of requests without answering. - server.setRoute('/fetch-request-a.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-b.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-c.js', (_req, res) => { - return responses.push(res); - }); - server.setRoute('/fetch-request-d.js', (_req, res) => { - return responses.push(res); - }); - const initialFetchResourcesRequested = Promise.all([ - server.waitForRequest('/fetch-request-a.js'), - server.waitForRequest('/fetch-request-b.js'), - server.waitForRequest('/fetch-request-c.js'), - ]); - const secondFetchResourceRequested = server.waitForRequest( - '/fetch-request-d.js' - ); + it('should wait for network idle to succeed navigation', async () => { + const {page, server} = getTestState(); - // Navigate to a page which loads immediately and then does a bunch of - // requests via javascript's fetch method. - const navigationPromise = page.goto( - server.PREFIX + '/networkidle.html', - { - waitUntil: 'networkidle0', - } - ); - // Track when the navigation gets completed. - let navigationFinished = false; - navigationPromise.then(() => { - return (navigationFinished = true); - }); + let responses: ServerResponse[] = []; + // Hold on to a bunch of requests without answering. + server.setRoute('/fetch-request-a.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-b.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-c.js', (_req, res) => { + return responses.push(res); + }); + server.setRoute('/fetch-request-d.js', (_req, res) => { + return responses.push(res); + }); + const initialFetchResourcesRequested = Promise.all([ + server.waitForRequest('/fetch-request-a.js'), + server.waitForRequest('/fetch-request-b.js'), + server.waitForRequest('/fetch-request-c.js'), + ]); + const secondFetchResourceRequested = server.waitForRequest( + '/fetch-request-d.js' + ); - // Wait for the page's 'load' event. - await new Promise(fulfill => { - return page.once('load', fulfill); - }); - expect(navigationFinished).toBe(false); + // Navigate to a page which loads immediately and then does a bunch of + // requests via javascript's fetch method. + const navigationPromise = page.goto(server.PREFIX + '/networkidle.html', { + waitUntil: 'networkidle0', + }); + // Track when the navigation gets completed. + let navigationFinished = false; + navigationPromise.then(() => { + return (navigationFinished = true); + }); - // Wait for the initial three resources to be requested. - await initialFetchResourcesRequested; + // Wait for the page's 'load' event. + await new Promise(fulfill => { + return page.once('load', fulfill); + }); + expect(navigationFinished).toBe(false); - // Expect navigation still to be not finished. - expect(navigationFinished).toBe(false); + // Wait for the initial three resources to be requested. + await initialFetchResourcesRequested; - // Respond to initial requests. - for (const response of responses) { - response.statusCode = 404; - response.end(`File not found`); - } + // Expect navigation still to be not finished. + expect(navigationFinished).toBe(false); - // Reset responses array - responses = []; + // Respond to initial requests. + for (const response of responses) { + response.statusCode = 404; + response.end(`File not found`); + } - // Wait for the second round to be requested. - await secondFetchResourceRequested; - // Expect navigation still to be not finished. - expect(navigationFinished).toBe(false); + // Reset responses array + responses = []; - // Respond to requests. - for (const response of responses) { - response.statusCode = 404; - response.end(`File not found`); - } + // Wait for the second round to be requested. + await secondFetchResourceRequested; + // Expect navigation still to be not finished. + expect(navigationFinished).toBe(false); - const response = (await navigationPromise)!; - // Expect navigation to succeed. - expect(response.ok()).toBe(true); + // Respond to requests. + for (const response of responses) { + response.statusCode = 404; + response.end(`File not found`); } - ); + + const response = (await navigationPromise)!; + // Expect navigation to succeed. + expect(response.ok()).toBe(true); + }); it('should not leak listeners during navigation', async () => { const {page, server} = getTestState(); @@ -461,38 +448,32 @@ describe('navigation', function () { process.removeListener('warning', warningHandler); expect(warning).toBe(null); }); - itFailsFirefox( - 'should navigate to dataURL and fire dataURL requests', - async () => { - const {page} = getTestState(); + it('should navigate to dataURL and fire dataURL requests', async () => { + const {page} = getTestState(); - const requests: HTTPRequest[] = []; - page.on('request', request => { - return !utils.isFavicon(request) && requests.push(request); - }); - const dataURL = 'data:text/html,
yo
'; - const response = (await page.goto(dataURL))!; - expect(response.status()).toBe(200); - expect(requests.length).toBe(1); - expect(requests[0]!.url()).toBe(dataURL); - } - ); - itFailsFirefox( - 'should navigate to URL with hash and fire requests without hash', - async () => { - const {page, server} = getTestState(); - - const requests: HTTPRequest[] = []; - page.on('request', request => { - return !utils.isFavicon(request) && requests.push(request); - }); - const response = (await page.goto(server.EMPTY_PAGE + '#hash'))!; - expect(response.status()).toBe(200); - expect(response.url()).toBe(server.EMPTY_PAGE); - expect(requests.length).toBe(1); - expect(requests[0]!.url()).toBe(server.EMPTY_PAGE); - } - ); + const requests: HTTPRequest[] = []; + page.on('request', request => { + return !utils.isFavicon(request) && requests.push(request); + }); + const dataURL = 'data:text/html,
yo
'; + const response = (await page.goto(dataURL))!; + expect(response.status()).toBe(200); + expect(requests.length).toBe(1); + expect(requests[0]!.url()).toBe(dataURL); + }); + it('should navigate to URL with hash and fire requests without hash', async () => { + const {page, server} = getTestState(); + + const requests: HTTPRequest[] = []; + page.on('request', request => { + return !utils.isFavicon(request) && requests.push(request); + }); + const response = (await page.goto(server.EMPTY_PAGE + '#hash'))!; + expect(response.status()).toBe(200); + expect(response.url()).toBe(server.EMPTY_PAGE); + expect(requests.length).toBe(1); + expect(requests[0]!.url()).toBe(server.EMPTY_PAGE); + }); it('should work with self requesting page', async () => { const {page, server} = getTestState(); @@ -512,7 +493,7 @@ describe('navigation', function () { } expect(error.message).toContain(url); }); - itFailsFirefox('should send referer', async () => { + it('should send referer', async () => { const {page, server} = getTestState(); const [request1, request2] = await Promise.all([ @@ -582,7 +563,7 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.EMPTY_PAGE + '#foobar'); }); - itFailsFirefox('should work with history.pushState()', async () => { + it('should work with history.pushState()', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -599,7 +580,7 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/wow.html'); }); - itFailsFirefox('should work with history.replaceState()', async () => { + it('should work with history.replaceState()', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -616,13 +597,11 @@ describe('navigation', function () { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/replaced.html'); }); - itFailsFirefox( - 'should work with DOM history.back()/history.forward()', - async () => { - const {page, server} = getTestState(); + it('should work with DOM history.back()/history.forward()', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - await page.setContent(` + await page.goto(server.EMPTY_PAGE); + await page.setContent(` back forward `); - expect(page.url()).toBe(server.PREFIX + '/second.html'); - const [backResponse] = await Promise.all([ - page.waitForNavigation(), - page.click('a#back'), - ]); - expect(backResponse).toBe(null); - expect(page.url()).toBe(server.PREFIX + '/first.html'); - const [forwardResponse] = await Promise.all([ - page.waitForNavigation(), - page.click('a#forward'), - ]); - expect(forwardResponse).toBe(null); - expect(page.url()).toBe(server.PREFIX + '/second.html'); - } - ); - itFailsFirefox( - 'should work when subframe issues window.stop()', - async () => { - const {page, server} = getTestState(); - - server.setRoute('/frames/style.css', () => {}); - const navigationPromise = page.goto( - server.PREFIX + '/frames/one-frame.html' - ); - const frame = await utils.waitEvent(page, 'frameattached'); - await new Promise(fulfill => { - page.on('framenavigated', f => { - if (f === frame) { - fulfill(); - } - }); + expect(page.url()).toBe(server.PREFIX + '/second.html'); + const [backResponse] = await Promise.all([ + page.waitForNavigation(), + page.click('a#back'), + ]); + expect(backResponse).toBe(null); + expect(page.url()).toBe(server.PREFIX + '/first.html'); + const [forwardResponse] = await Promise.all([ + page.waitForNavigation(), + page.click('a#forward'), + ]); + expect(forwardResponse).toBe(null); + expect(page.url()).toBe(server.PREFIX + '/second.html'); + }); + it('should work when subframe issues window.stop()', async () => { + const {page, server} = getTestState(); + + server.setRoute('/frames/style.css', () => {}); + const navigationPromise = page.goto( + server.PREFIX + '/frames/one-frame.html' + ); + const frame = await utils.waitEvent(page, 'frameattached'); + await new Promise(fulfill => { + page.on('framenavigated', f => { + if (f === frame) { + fulfill(); + } }); - await Promise.all([ - frame.evaluate(() => { - return window.stop(); - }), - navigationPromise, - ]); - } - ); + }); + await Promise.all([ + frame.evaluate(() => { + return window.stop(); + }), + navigationPromise, + ]); + }); }); describe('Page.goBack', function () { @@ -692,7 +667,7 @@ describe('navigation', function () { response = (await page.goForward())!; expect(response).toBe(null); }); - itFailsFirefox('should work with HistoryAPI', async () => { + it('should work with HistoryAPI', async () => { const {page, server} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -711,7 +686,7 @@ describe('navigation', function () { }); }); - describeFailsFirefox('Frame.goto', function () { + describe('Frame.goto', function () { it('should navigate subframes', async () => { const {page, server} = getTestState(); @@ -776,7 +751,7 @@ describe('navigation', function () { }); }); - describeFailsFirefox('Frame.waitForNavigation', function () { + describe('Frame.waitForNavigation', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/network.spec.ts b/test/src/network.spec.ts index 9faad3bf970fe..b71b0187f7585 100644 --- a/test/src/network.spec.ts +++ b/test/src/network.spec.ts @@ -22,14 +22,11 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - describeFailsFirefox, - itChromeOnly, - itFirefoxOnly, } from './mocha-utils.js'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {HTTPResponse} from '../../lib/cjs/puppeteer/common/HTTPResponse.js'; import {ServerResponse} from 'http'; +import {it} from './mocha-utils.js'; describe('network', function () { setupTestBrowserHooks(); @@ -83,7 +80,7 @@ describe('network', function () { expect(requests.length).toBe(1); expect(requests[0]!.frame()).toBe(page.mainFrame()); }); - itFailsFirefox('should work for subframe navigation request', async () => { + it('should work for subframe navigation request', async () => { const {page, server} = getTestState(); (await page.goto(server.EMPTY_PAGE))!; @@ -115,13 +112,13 @@ describe('network', function () { }); describe('Request.headers', function () { - itChromeOnly('should define Chrome as user agent header', async () => { + it('should define Chrome as user agent header', async () => { const {page, server} = getTestState(); const response = (await page.goto(server.EMPTY_PAGE))!; expect(response.request().headers()['user-agent']).toContain('Chrome'); }); - itFirefoxOnly('should define Firefox as user agent header', async () => { + it('should define Firefox as user agent header', async () => { const {page, server} = getTestState(); const response = (await page.goto(server.EMPTY_PAGE))!; @@ -142,7 +139,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Request.initiator', () => { + describe('Request.initiator', () => { it('should return the initiator', async () => { const {page, server} = getTestState(); @@ -187,7 +184,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.fromCache', function () { + describe('Response.fromCache', function () { it('should return |false| for non-cached content', async () => { const {page, server} = getTestState(); @@ -218,7 +215,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.fromServiceWorker', function () { + describe('Response.fromServiceWorker', function () { it('should return |false| for non-service-worker content', async () => { const {page, server} = getTestState(); @@ -253,7 +250,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Request.postData', function () { + describe('Request.postData', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -284,7 +281,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.text', function () { + describe('Response.text', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -367,7 +364,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.json', function () { + describe('Response.json', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -376,7 +373,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.buffer', function () { + describe('Response.buffer', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -461,7 +458,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Response.timing', function () { + describe('Response.timing', function () { it('returns timing information', async () => { const {page, server} = getTestState(); const responses: HTTPResponse[] = []; @@ -474,7 +471,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Network Events', function () { + describe('Network Events', function () { it('Page.Events.Request', async () => { const {page, server} = getTestState(); @@ -624,7 +621,7 @@ describe('network', function () { }); describe('Request.isNavigationRequest', () => { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); const requests = new Map(); @@ -639,7 +636,7 @@ describe('network', function () { expect(requests.get('script.js').isNavigationRequest()).toBe(false); expect(requests.get('style.css').isNavigationRequest()).toBe(false); }); - itFailsFirefox('should work with request interception', async () => { + it('should work with request interception', async () => { const {page, server} = getTestState(); const requests = new Map(); @@ -656,7 +653,7 @@ describe('network', function () { expect(requests.get('script.js').isNavigationRequest()).toBe(false); expect(requests.get('style.css').isNavigationRequest()).toBe(false); }); - itFailsFirefox('should work when navigating to image', async () => { + it('should work when navigating to image', async () => { const {page, server} = getTestState(); const requests: HTTPRequest[] = []; @@ -668,7 +665,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Page.setExtraHTTPHeaders', function () { + describe('Page.setExtraHTTPHeaders', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -697,7 +694,7 @@ describe('network', function () { }); }); - describeFailsFirefox('Page.authenticate', function () { + describe('Page.authenticate', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -794,7 +791,7 @@ describe('network', function () { }); }); - describeFailsFirefox('raw network headers', async () => { + describe('raw network headers', async () => { it('Same-origin set-cookie navigation', async () => { const {page, server} = getTestState(); diff --git a/test/src/oopif.spec.ts b/test/src/oopif.spec.ts index 17762636d2338..174d3c5ae6920 100644 --- a/test/src/oopif.spec.ts +++ b/test/src/oopif.spec.ts @@ -16,18 +16,15 @@ import utils from './utils.js'; import expect from 'expect'; -import { - getTestState, - describeChromeOnly, - itFailsFirefox, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import { Browser, BrowserContext, } from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('OOPIF', function () { +describe('OOPIF', function () { /* We use a special browser for this test as we need the --site-per-process flag */ let browser: Browser; let context: BrowserContext; @@ -207,6 +204,7 @@ describeChromeOnly('OOPIF', function () { await utils.navigateFrame(page, 'frame1', server.EMPTY_PAGE); expect(frame.url()).toBe(server.EMPTY_PAGE); }); + it('should support evaluating in oop iframes', async () => { const {server} = getTestState(); @@ -420,7 +418,7 @@ describeChromeOnly('OOPIF', function () { browser1.disconnect(); }); - itFailsFirefox('should support lazy OOP frames', async () => { + it('should support lazy OOP frames', async () => { const {server} = getTestState(); await page.goto(server.PREFIX + '/lazy-oopif-frame.html'); diff --git a/test/src/page.spec.ts b/test/src/page.spec.ts index 1f7510e464878..3d59994a12a36 100644 --- a/test/src/page.spec.ts +++ b/test/src/page.spec.ts @@ -22,13 +22,12 @@ import {CDPSession} from '../../lib/cjs/puppeteer/common/Connection.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import {Metrics, Page} from '../../lib/cjs/puppeteer/common/Page.js'; import { - describeFailsFirefox, getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils, {attachFrame, waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; describe('Page', function () { setupTestBrowserHooks(); @@ -59,7 +58,7 @@ describe('Page', function () { await newPage.close(); expect(await browser.pages()).not.toContain(newPage); }); - itFailsFirefox('should run beforeunload if asked for', async () => { + it('should run beforeunload if asked for', async () => { const {context, server, isChrome} = getTestState(); const newPage = await context.newPage(); @@ -79,7 +78,7 @@ describe('Page', function () { await dialog.accept(); await pageClosingPromise; }); - itFailsFirefox('should *not* run beforeunload by default', async () => { + it('should *not* run beforeunload by default', async () => { const {context, server} = getTestState(); const newPage = await context.newPage(); @@ -97,7 +96,7 @@ describe('Page', function () { await newPage.close(); expect(newPage.isClosed()).toBe(true); }); - itFailsFirefox('should terminate network waiters', async () => { + it('should terminate network waiters', async () => { const {context, server} = getTestState(); const newPage = await context.newPage(); @@ -182,7 +181,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.error', function () { + describe('Page.Events.error', function () { it('should throw when page crashes', async () => { const {page} = getTestState(); @@ -196,7 +195,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.Popup', function () { + describe('Page.Events.Popup', function () { it('should work', async () => { const {page} = getTestState(); @@ -354,7 +353,7 @@ describe('Page', function () { await page.goto(server.EMPTY_PAGE); expect(await getPermission(page, 'geolocation')).toBe('prompt'); }); - itFailsFirefox('should deny permission when not listed', async () => { + it('should deny permission when not listed', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -374,14 +373,14 @@ describe('Page', function () { }); expect(error.message).toBe('Unknown permission: foo'); }); - itFailsFirefox('should grant permission when listed', async () => { + it('should grant permission when listed', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); await context.overridePermissions(server.EMPTY_PAGE, ['geolocation']); expect(await getPermission(page, 'geolocation')).toBe('granted'); }); - itFailsFirefox('should reset permissions', async () => { + it('should reset permissions', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -390,7 +389,7 @@ describe('Page', function () { await context.clearPermissionOverrides(); expect(await getPermission(page, 'geolocation')).toBe('prompt'); }); - itFailsFirefox('should trigger permission onchange', async () => { + it('should trigger permission onchange', async () => { const {page, server, context, isHeadless} = getTestState(); // TODO: re-enable this test in headful once crbug.com/1324480 rolls out. @@ -434,33 +433,30 @@ describe('Page', function () { }) ).toEqual(['prompt', 'denied', 'granted', 'prompt']); }); - itFailsFirefox( - 'should isolate permissions between browser contexts', - async () => { - const {page, server, context, browser} = getTestState(); - - await page.goto(server.EMPTY_PAGE); - const otherContext = await browser.createIncognitoBrowserContext(); - const otherPage = await otherContext.newPage(); - await otherPage.goto(server.EMPTY_PAGE); - expect(await getPermission(page, 'geolocation')).toBe('prompt'); - expect(await getPermission(otherPage, 'geolocation')).toBe('prompt'); - - await context.overridePermissions(server.EMPTY_PAGE, []); - await otherContext.overridePermissions(server.EMPTY_PAGE, [ - 'geolocation', - ]); - expect(await getPermission(page, 'geolocation')).toBe('denied'); - expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + it('should isolate permissions between browser contexts', async () => { + const {page, server, context, browser} = getTestState(); - await context.clearPermissionOverrides(); - expect(await getPermission(page, 'geolocation')).toBe('prompt'); - expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + await page.goto(server.EMPTY_PAGE); + const otherContext = await browser.createIncognitoBrowserContext(); + const otherPage = await otherContext.newPage(); + await otherPage.goto(server.EMPTY_PAGE); + expect(await getPermission(page, 'geolocation')).toBe('prompt'); + expect(await getPermission(otherPage, 'geolocation')).toBe('prompt'); - await otherContext.close(); - } - ); - itFailsFirefox('should grant persistent-storage', async () => { + await context.overridePermissions(server.EMPTY_PAGE, []); + await otherContext.overridePermissions(server.EMPTY_PAGE, [ + 'geolocation', + ]); + expect(await getPermission(page, 'geolocation')).toBe('denied'); + expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + + await context.clearPermissionOverrides(); + expect(await getPermission(page, 'geolocation')).toBe('prompt'); + expect(await getPermission(otherPage, 'geolocation')).toBe('granted'); + + await otherContext.close(); + }); + it('should grant persistent-storage', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -475,7 +471,7 @@ describe('Page', function () { }); describe('Page.setGeolocation', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server, context} = getTestState(); await context.overridePermissions(server.PREFIX, ['geolocation']); @@ -509,7 +505,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setOfflineMode', function () { + describe('Page.setOfflineMode', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -547,7 +543,7 @@ describe('Page', function () { }); describe('ExecutionContext.queryObjects', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page} = getTestState(); // Instantiate an object @@ -567,7 +563,7 @@ describe('Page', function () { }, objectsHandle); expect(values).toEqual(['hello', 'world']); }); - itFailsFirefox('should work for non-blank page', async () => { + it('should work for non-blank page', async () => { const {page, server} = getTestState(); // Instantiate an object @@ -613,7 +609,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.Console', function () { + describe('Page.Events.Console', function () { it('should work', async () => { const {page} = getTestState(); @@ -802,7 +798,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.metrics', function () { + describe('Page.metrics', function () { it('should get metrics from a page', async () => { const {page} = getTestState(); @@ -1108,7 +1104,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.exposeFunction', function () { + describe('Page.exposeFunction', function () { it('should work', async () => { const {page} = getTestState(); @@ -1266,7 +1262,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.Events.PageError', function () { + describe('Page.Events.PageError', function () { it('should fire', async () => { const {page, server} = getTestState(); @@ -1329,7 +1325,7 @@ describe('Page', function () { }) ).toContain('iPhone'); }); - itFailsFirefox('should work with additional userAgentMetdata', async () => { + it('should work with additional userAgentMetdata', async () => { const {page, server} = getTestState(); await page.setUserAgent('MockBrowser', { @@ -1496,7 +1492,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setBypassCSP', function () { + describe('Page.setBypassCSP', function () { it('should bypass CSP meta tag', async () => { const {page, server} = getTestState(); @@ -1879,21 +1875,18 @@ describe('Page', function () { ).toBe('rgb(0, 128, 0)'); }); - itFailsFirefox( - 'should throw when added with content to the CSP page', - async () => { - const {page, server} = getTestState(); + it('should throw when added with content to the CSP page', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/csp.html'); - let error!: Error; - await page - .addStyleTag({content: 'body { background-color: green; }'}) - .catch(error_ => { - return (error = error_); - }); - expect(error).toBeTruthy(); - } - ); + await page.goto(server.PREFIX + '/csp.html'); + let error!: Error; + await page + .addStyleTag({content: 'body { background-color: green; }'}) + .catch(error_ => { + return (error = error_); + }); + expect(error).toBeTruthy(); + }); it('should throw when added with URL to the CSP page', async () => { const {page, server} = getTestState(); @@ -1921,7 +1914,7 @@ describe('Page', function () { }); }); - describeFailsFirefox('Page.setJavaScriptEnabled', function () { + describe('Page.setJavaScriptEnabled', function () { it('should work', async () => { const {page} = getTestState(); @@ -1962,23 +1955,20 @@ describe('Page', function () { ]); expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); }); - itFailsFirefox( - 'should stay disabled when toggling request interception on/off', - async () => { - const {page, server} = getTestState(); + it('should stay disabled when toggling request interception on/off', async () => { + const {page, server} = getTestState(); - await page.setCacheEnabled(false); - await page.setRequestInterception(true); - await page.setRequestInterception(false); + await page.setCacheEnabled(false); + await page.setRequestInterception(true); + await page.setRequestInterception(false); - await page.goto(server.PREFIX + '/cached/one-style.html'); - const [nonCachedRequest] = await Promise.all([ - server.waitForRequest('/cached/one-style.html'), - page.reload(), - ]); - expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); - } - ); + await page.goto(server.PREFIX + '/cached/one-style.html'); + const [nonCachedRequest] = await Promise.all([ + server.waitForRequest('/cached/one-style.html'), + page.reload(), + ]); + expect(nonCachedRequest.headers['if-modified-since']).toBe(undefined); + }); }); describe('printing to PDF', function () { @@ -2220,33 +2210,30 @@ describe('Page', function () { expect(error.message).toContain('Values must be strings'); }); // @see https://github.com/puppeteer/puppeteer/issues/3327 - itFailsFirefox( - 'should work when re-defining top-level Event class', - async () => { - const {page, server} = getTestState(); + it('should work when re-defining top-level Event class', async () => { + const {page, server} = getTestState(); - await page.goto(server.PREFIX + '/input/select.html'); + await page.goto(server.PREFIX + '/input/select.html'); + await page.evaluate(() => { + // @ts-expect-error Expected. + return (window.Event = undefined); + }); + await page.select('select', 'blue'); + expect( await page.evaluate(() => { - // @ts-expect-error Expected. - return (window.Event = undefined); - }); - await page.select('select', 'blue'); - expect( - await page.evaluate(() => { - return (globalThis as any).result.onInput; - }) - ).toEqual(['blue']); - expect( - await page.evaluate(() => { - return (globalThis as any).result.onChange; - }) - ).toEqual(['blue']); - } - ); + return (globalThis as any).result.onInput; + }) + ).toEqual(['blue']); + expect( + await page.evaluate(() => { + return (globalThis as any).result.onChange; + }) + ).toEqual(['blue']); + }); }); describe('Page.Events.Close', function () { - itFailsFirefox('should work with window.close', async () => { + it('should work with window.close', async () => { const {page, context} = getTestState(); const newPagePromise = new Promise(fulfill => { diff --git a/test/src/proxy.spec.ts b/test/src/proxy.spec.ts index eb71905355d34..9aac24e4bcbdf 100644 --- a/test/src/proxy.spec.ts +++ b/test/src/proxy.spec.ts @@ -17,15 +17,12 @@ import expect from 'expect'; import http from 'http'; import os from 'os'; -import { - getTestState, - describeFailsFirefox, - itFailsWindows, -} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import type {Server, IncomingMessage, ServerResponse} from 'http'; import type {Browser} from '../../lib/cjs/puppeteer/common/Browser.js'; import type {AddressInfo} from 'net'; import {TestServer} from '../../utils/testserver/lib/index.js'; +import {it} from './mocha-utils.js'; let HOSTNAME = os.hostname(); @@ -53,7 +50,7 @@ function getEmptyPageUrl(server: TestServer): string { return `http://${HOSTNAME}:${server.PORT}${emptyPagePath}`; } -describeFailsFirefox('request proxy', () => { +describe('request proxy', () => { let browser: Browser; let proxiedRequestUrls: string[]; let proxyServer: Server; @@ -194,28 +191,25 @@ describeFailsFirefox('request proxy', () => { /** * See issues #7873, #7719, and #7698. */ - itFailsWindows( - 'should proxy requests when configured at context level', - async () => { - const {puppeteer, defaultBrowserOptions, server} = getTestState(); - const emptyPageUrl = getEmptyPageUrl(server); - - browser = await puppeteer.launch({ - ...defaultBrowserOptions, - args: defaultArgs, - }); - - const context = await browser.createIncognitoBrowserContext({ - proxyServer: proxyServerUrl, - }); - const page = await context.newPage(); - const response = (await page.goto(emptyPageUrl))!; - - expect(response.ok()).toBe(true); - - expect(proxiedRequestUrls).toEqual([emptyPageUrl]); - } - ); + it('should proxy requests when configured at context level', async () => { + const {puppeteer, defaultBrowserOptions, server} = getTestState(); + const emptyPageUrl = getEmptyPageUrl(server); + + browser = await puppeteer.launch({ + ...defaultBrowserOptions, + args: defaultArgs, + }); + + const context = await browser.createIncognitoBrowserContext({ + proxyServer: proxyServerUrl, + }); + const page = await context.newPage(); + const response = (await page.goto(emptyPageUrl))!; + + expect(response.ok()).toBe(true); + + expect(proxiedRequestUrls).toEqual([emptyPageUrl]); + }); it('should respect proxy bypass list when configured at context level', async () => { const {puppeteer, defaultBrowserOptions, server} = getTestState(); diff --git a/test/src/queryhandler.spec.ts b/test/src/queryhandler.spec.ts index 30e5d69254d52..921fbb7d6b2d9 100644 --- a/test/src/queryhandler.spec.ts +++ b/test/src/queryhandler.spec.ts @@ -20,6 +20,7 @@ import { setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Query handler tests', function () { setupTestBrowserHooks(); diff --git a/test/src/queryselector.spec.ts b/test/src/queryselector.spec.ts index 7eeab8e44e7f6..8c94a60e74c33 100644 --- a/test/src/queryselector.spec.ts +++ b/test/src/queryselector.spec.ts @@ -20,6 +20,7 @@ import { setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('querySelector', function () { setupTestBrowserHooks(); diff --git a/test/src/requestinterception-experimental.spec.ts b/test/src/requestinterception-experimental.spec.ts index 687588237d52b..02971d202340b 100644 --- a/test/src/requestinterception-experimental.spec.ts +++ b/test/src/requestinterception-experimental.spec.ts @@ -22,7 +22,6 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import { @@ -30,11 +29,12 @@ import { HTTPRequest, InterceptResolutionAction, } from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; +import {it} from './mocha-utils.js'; describe('request interception', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('Page.setRequestInterception', function () { + describe('Page.setRequestInterception', function () { const expectedActions: ActionResult[] = ['abort', 'continue', 'respond']; while (expectedActions.length > 0) { const expectedAction = expectedActions.pop(); @@ -709,7 +709,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.continue', function () { + describe('Request.continue', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -805,7 +805,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.respond', function () { + describe('Request.respond', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/requestinterception.spec.ts b/test/src/requestinterception.spec.ts index bb7e0c03dd644..a1abc3ab5a921 100644 --- a/test/src/requestinterception.spec.ts +++ b/test/src/requestinterception.spec.ts @@ -22,15 +22,15 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; import {HTTPRequest} from '../../lib/cjs/puppeteer/common/HTTPRequest.js'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; +import {it} from './mocha-utils.js'; describe('request interception', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); - describeFailsFirefox('Page.setRequestInterception', function () { + describe('Page.setRequestInterception', function () { it('should intercept', async () => { const {page, server} = getTestState(); @@ -623,7 +623,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.continue', function () { + describe('Request.continue', function () { it('should work', async () => { const {page, server} = getTestState(); @@ -739,7 +739,7 @@ describe('request interception', function () { }); }); - describeFailsFirefox('Request.respond', function () { + describe('Request.respond', function () { it('should work', async () => { const {page, server} = getTestState(); diff --git a/test/src/screenshot.spec.ts b/test/src/screenshot.spec.ts index 85ef96b203af6..eefeda4bf61ce 100644 --- a/test/src/screenshot.spec.ts +++ b/test/src/screenshot.spec.ts @@ -19,17 +19,15 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - itFailsFirefox, - itHeadfulOnly, - itChromeOnly, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; describe('Screenshots', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); describe('Page.screenshot', function () { - itFailsFirefox('should work', async () => { + it('should work', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -37,7 +35,7 @@ describe('Screenshots', function () { const screenshot = await page.screenshot(); expect(screenshot).toBeGolden('screenshot-sanity.png'); }); - itFailsFirefox('should clip rect', async () => { + it('should clip rect', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -52,7 +50,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-rect.png'); }); - itFailsFirefox('should use scale for clip', async () => { + it('should use scale for clip', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -68,23 +66,20 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-rect-scale2.png'); }); - itFailsFirefox( - 'should get screenshot bigger than the viewport', - async () => { - const {page, server} = getTestState(); - await page.setViewport({width: 50, height: 50}); - await page.goto(server.PREFIX + '/grid.html'); - const screenshot = await page.screenshot({ - clip: { - x: 25, - y: 25, - width: 100, - height: 100, - }, - }); - expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); - } - ); + it('should get screenshot bigger than the viewport', async () => { + const {page, server} = getTestState(); + await page.setViewport({width: 50, height: 50}); + await page.goto(server.PREFIX + '/grid.html'); + const screenshot = await page.screenshot({ + clip: { + x: 25, + y: 25, + width: 100, + height: 100, + }, + }); + expect(screenshot).toBeGolden('screenshot-offscreen-clip.png'); + }); it('should run in parallel', async () => { const {page, server} = getTestState(); @@ -106,7 +101,7 @@ describe('Screenshots', function () { const screenshots = await Promise.all(promises); expect(screenshots[1]!).toBeGolden('grid-cell-1.png'); }); - itFailsFirefox('should take fullPage screenshots', async () => { + it('should take fullPage screenshots', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -147,7 +142,7 @@ describe('Screenshots', function () { }) ); }); - itFailsFirefox('should allow transparency', async () => { + it('should allow transparency', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -155,7 +150,7 @@ describe('Screenshots', function () { const screenshot = await page.screenshot({omitBackground: true}); expect(screenshot).toBeGolden('transparent.png'); }); - itFailsFirefox('should render white background on jpeg file', async () => { + it('should render white background on jpeg file', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -166,7 +161,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('white.jpg'); }); - itFailsFirefox('should work with webp', async () => { + it('should work with webp', async () => { const {page, server} = getTestState(); await page.setViewport({width: 100, height: 100}); @@ -190,7 +185,7 @@ describe('Screenshots', function () { }); expect(screenshot).toBeGolden('screenshot-clip-odd-size.png'); }); - itFailsFirefox('should return base64', async () => { + it('should return base64', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -206,7 +201,7 @@ describe('Screenshots', function () { 'screenshot-sanity.png' ); }); - itHeadfulOnly('should work in "fromSurface: false" mode', async () => { + it('should work in "fromSurface: false" mode', async () => { const {page, server} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -231,7 +226,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-bounding-box.png'); }); - itChromeOnly('should work with a null viewport', async () => { + it('should work with a null viewport', async () => { const {defaultBrowserOptions, puppeteer, server} = getTestState(); const browser = await puppeteer.launch({ @@ -271,14 +266,12 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-padding-border.png'); }); - itFailsFirefox( - 'should capture full element when larger than viewport', - async () => { - const {page} = getTestState(); + it('should capture full element when larger than viewport', async () => { + const {page} = getTestState(); - await page.setViewport({width: 500, height: 500}); + await page.setViewport({width: 500, height: 500}); - await page.setContent(` + await page.setContent(` something above
`); - const elementHandle = (await page.$('div.to-screenshot'))!; - const screenshot = await elementHandle.screenshot(); - expect(screenshot).toBeGolden( - 'screenshot-element-larger-than-viewport.png' - ); + const elementHandle = (await page.$('div.to-screenshot'))!; + const screenshot = await elementHandle.screenshot(); + expect(screenshot).toBeGolden( + 'screenshot-element-larger-than-viewport.png' + ); - expect( - await page.evaluate(() => { - return { - w: window.innerWidth, - h: window.innerHeight, - }; - }) - ).toEqual({w: 500, h: 500}); - } - ); + expect( + await page.evaluate(() => { + return { + w: window.innerWidth, + h: window.innerHeight, + }; + }) + ).toEqual({w: 500, h: 500}); + }); it('should scroll element into view', async () => { const {page} = getTestState(); @@ -336,7 +328,7 @@ describe('Screenshots', function () { 'screenshot-element-scrolled-into-view.png' ); }); - itFailsFirefox('should work with a rotated element', async () => { + it('should work with a rotated element', async () => { const {page} = getTestState(); await page.setViewport({width: 500, height: 500}); @@ -351,7 +343,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-rotate.png'); }); - itFailsFirefox('should fail to screenshot a detached element', async () => { + it('should fail to screenshot a detached element', async () => { const {page} = getTestState(); await page.setContent('

remove this

'); @@ -386,7 +378,7 @@ describe('Screenshots', function () { const screenshot = await elementHandle.screenshot(); expect(screenshot).toBeGolden('screenshot-element-fractional.png'); }); - itFailsFirefox('should work for an element with an offset', async () => { + it('should work for an element with an offset', async () => { const {page} = getTestState(); await page.setContent( diff --git a/test/src/target.spec.ts b/test/src/target.spec.ts index a67b31f0ee866..96fbc5fe359fd 100644 --- a/test/src/target.spec.ts +++ b/test/src/target.spec.ts @@ -20,11 +20,12 @@ import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; import {Target} from '../../lib/cjs/puppeteer/common/Target.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import utils from './utils.js'; +import {it} from './mocha-utils.js'; + const {waitEvent} = utils; describe('Target', function () { @@ -79,7 +80,7 @@ describe('Target', function () { ).toBe('Hello world'); expect(await originalPage.$('body')).toBeTruthy(); }); - itFailsFirefox('should be able to use async waitForTarget', async () => { + it('should be able to use async waitForTarget', async () => { const {page, server, context} = getTestState(); const [otherPage] = await Promise.all([ @@ -101,89 +102,83 @@ describe('Target', function () { ); expect(page).not.toEqual(otherPage); }); - itFailsFirefox( - 'should report when a new page is created and closed', - async () => { - const {page, server, context} = getTestState(); + it('should report when a new page is created and closed', async () => { + const {page, server, context} = getTestState(); - const [otherPage] = await Promise.all([ - context - .waitForTarget(target => { - return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; - }) - .then(target => { - return target.page(); - }), - page.evaluate((url: string) => { - return window.open(url); - }, server.CROSS_PROCESS_PREFIX + '/empty.html'), - ]); - expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); - expect( - await otherPage!.evaluate(() => { - return ['Hello', 'world'].join(' '); + const [otherPage] = await Promise.all([ + context + .waitForTarget(target => { + return target.url() === server.CROSS_PROCESS_PREFIX + '/empty.html'; }) - ).toBe('Hello world'); - expect(await otherPage!.$('body')).toBeTruthy(); + .then(target => { + return target.page(); + }), + page.evaluate((url: string) => { + return window.open(url); + }, server.CROSS_PROCESS_PREFIX + '/empty.html'), + ]); + expect(otherPage!.url()).toContain(server.CROSS_PROCESS_PREFIX); + expect( + await otherPage!.evaluate(() => { + return ['Hello', 'world'].join(' '); + }) + ).toBe('Hello world'); + expect(await otherPage!.$('body')).toBeTruthy(); - let allPages = await context.pages(); - expect(allPages).toContain(page); - expect(allPages).toContain(otherPage); + let allPages = await context.pages(); + expect(allPages).toContain(page); + expect(allPages).toContain(otherPage); - const closePagePromise = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target.page()); - }); + const closePagePromise = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target.page()); }); - await otherPage!.close(); - expect(await closePagePromise).toBe(otherPage); + }); + await otherPage!.close(); + expect(await closePagePromise).toBe(otherPage); - allPages = (await Promise.all( - context.targets().map(target => { - return target.page(); - }) - )) as Page[]; - expect(allPages).toContain(page); - expect(allPages).not.toContain(otherPage); - } - ); - itFailsFirefox( - 'should report when a service worker is created and destroyed', - async () => { - const {page, server, context} = getTestState(); + allPages = (await Promise.all( + context.targets().map(target => { + return target.page(); + }) + )) as Page[]; + expect(allPages).toContain(page); + expect(allPages).not.toContain(otherPage); + }); + it('should report when a service worker is created and destroyed', async () => { + const {page, server, context} = getTestState(); - await page.goto(server.EMPTY_PAGE); - const createdTarget = new Promise(fulfill => { - return context.once('targetcreated', target => { - return fulfill(target); - }); + await page.goto(server.EMPTY_PAGE); + const createdTarget = new Promise(fulfill => { + return context.once('targetcreated', target => { + return fulfill(target); }); + }); - await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); + await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); - expect((await createdTarget).type()).toBe('service_worker'); - expect((await createdTarget).url()).toBe( - server.PREFIX + '/serviceworkers/empty/sw.js' - ); + expect((await createdTarget).type()).toBe('service_worker'); + expect((await createdTarget).url()).toBe( + server.PREFIX + '/serviceworkers/empty/sw.js' + ); - const destroyedTarget = new Promise(fulfill => { - return context.once('targetdestroyed', target => { - return fulfill(target); - }); + const destroyedTarget = new Promise(fulfill => { + return context.once('targetdestroyed', target => { + return fulfill(target); }); - await page.evaluate(() => { - return ( - globalThis as unknown as { - registrationPromise: Promise<{unregister: () => void}>; - } - ).registrationPromise.then((registration: any) => { - return registration.unregister(); - }); + }); + await page.evaluate(() => { + return ( + globalThis as unknown as { + registrationPromise: Promise<{unregister: () => void}>; + } + ).registrationPromise.then((registration: any) => { + return registration.unregister(); }); - expect(await destroyedTarget).toBe(await createdTarget); - } - ); - itFailsFirefox('should create a worker from a service worker', async () => { + }); + expect(await destroyedTarget).toBe(await createdTarget); + }); + it('should create a worker from a service worker', async () => { const {page, server, context} = getTestState(); await page.goto(server.PREFIX + '/serviceworkers/empty/sw.html'); @@ -198,7 +193,7 @@ describe('Target', function () { }) ).toBe('[object ServiceWorkerGlobalScope]'); }); - itFailsFirefox('should create a worker from a shared worker', async () => { + it('should create a worker from a shared worker', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -215,7 +210,7 @@ describe('Target', function () { }) ).toBe('[object SharedWorkerGlobalScope]'); }); - itFailsFirefox('should report when a target url changes', async () => { + it('should report when a target url changes', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -235,7 +230,7 @@ describe('Target', function () { await page.goto(server.EMPTY_PAGE); expect((await changedTarget).url()).toBe(server.EMPTY_PAGE); }); - itFailsFirefox('should not report uninitialized pages', async () => { + it('should not report uninitialized pages', async () => { const {context} = getTestState(); let targetChanged = false; @@ -268,37 +263,34 @@ describe('Target', function () { expect(targetChanged).toBe(false); context.removeListener('targetchanged', listener); }); - itFailsFirefox( - 'should not crash while redirecting if original request was missed', - async () => { - const {page, server, context} = getTestState(); + it('should not crash while redirecting if original request was missed', async () => { + const {page, server, context} = getTestState(); - let serverResponse!: ServerResponse; - server.setRoute('/one-style.css', (_req, res) => { - return (serverResponse = res); - }); - // Open a new page. Use window.open to connect to the page later. - await Promise.all([ - page.evaluate((url: string) => { - return window.open(url); - }, server.PREFIX + '/one-style.html'), - server.waitForRequest('/one-style.css'), - ]); - // Connect to the opened page. - const target = await context.waitForTarget(target => { - return target.url().includes('one-style.html'); - }); - const newPage = (await target.page())!; - // Issue a redirect. - serverResponse.writeHead(302, {location: '/injectedstyle.css'}); - serverResponse.end(); - // Wait for the new page to load. - await waitEvent(newPage, 'load'); - // Cleanup. - await newPage.close(); - } - ); - itFailsFirefox('should have an opener', async () => { + let serverResponse!: ServerResponse; + server.setRoute('/one-style.css', (_req, res) => { + return (serverResponse = res); + }); + // Open a new page. Use window.open to connect to the page later. + await Promise.all([ + page.evaluate((url: string) => { + return window.open(url); + }, server.PREFIX + '/one-style.html'), + server.waitForRequest('/one-style.css'), + ]); + // Connect to the opened page. + const target = await context.waitForTarget(target => { + return target.url().includes('one-style.html'); + }); + const newPage = (await target.page())!; + // Issue a redirect. + serverResponse.writeHead(302, {location: '/injectedstyle.css'}); + serverResponse.end(); + // Wait for the new page to load. + await waitEvent(newPage, 'load'); + // Cleanup. + await newPage.close(); + }); + it('should have an opener', async () => { const {page, server, context} = getTestState(); await page.goto(server.EMPTY_PAGE); @@ -318,7 +310,7 @@ describe('Target', function () { }); describe('Browser.waitForTarget', () => { - itFailsFirefox('should wait for a target', async () => { + it('should wait for a target', async () => { const {browser, puppeteer, server} = getTestState(); let resolved = false; diff --git a/test/src/touchscreen.spec.ts b/test/src/touchscreen.spec.ts index f32a2158ac21d..1299267473280 100644 --- a/test/src/touchscreen.spec.ts +++ b/test/src/touchscreen.spec.ts @@ -19,10 +19,10 @@ import { getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, - describeFailsFirefox, } from './mocha-utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Touchscreen', function () { +describe('Touchscreen', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); diff --git a/test/src/tracing.spec.ts b/test/src/tracing.spec.ts index 2a7a4675cbc20..27b1ae50837c8 100644 --- a/test/src/tracing.spec.ts +++ b/test/src/tracing.spec.ts @@ -17,11 +17,12 @@ import fs from 'fs'; import path from 'path'; import expect from 'expect'; -import {getTestState, describeChromeOnly} from './mocha-utils.js'; +import {getTestState} from './mocha-utils.js'; import {Browser} from '../../lib/cjs/puppeteer/common/Browser.js'; import {Page} from '../../lib/cjs/puppeteer/common/Page.js'; +import {it} from './mocha-utils.js'; -describeChromeOnly('Tracing', function () { +describe('Tracing', function () { let outputFile!: string; let browser!: Browser; let page!: Page; diff --git a/test/src/waittask.spec.ts b/test/src/waittask.spec.ts index 3ac049d418ee2..0988b5ac5b97d 100644 --- a/test/src/waittask.spec.ts +++ b/test/src/waittask.spec.ts @@ -18,11 +18,11 @@ import expect from 'expect'; import {isErrorLike} from '../../lib/cjs/puppeteer/util/ErrorLike.js'; import { getTestState, - itFailsFirefox, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import {attachFrame, detachFrame} from './utils.js'; +import {it} from './mocha-utils.js'; describe('waittask specs', function () { setupTestBrowserHooks(); @@ -188,7 +188,7 @@ describe('waittask specs', function () { }); await watchdog; }); - itFailsFirefox('should work with strict CSP policy', async () => { + it('should work with strict CSP policy', async () => { const {page, server} = getTestState(); server.setCSP('/empty.html', 'script-src ' + server.PREFIX); @@ -421,7 +421,7 @@ describe('waittask specs', function () { await frame.waitForSelector('div'); }); - itFailsFirefox('should work with removed MutationObserver', async () => { + it('should work with removed MutationObserver', async () => { const {page} = getTestState(); await page.evaluate(() => { @@ -465,23 +465,20 @@ describe('waittask specs', function () { await watchdog; }); - itFailsFirefox( - 'Page.waitForSelector is shortcut for main frame', - async () => { - const {page, server} = getTestState(); + it('Page.waitForSelector is shortcut for main frame', async () => { + const {page, server} = getTestState(); - await page.goto(server.EMPTY_PAGE); - await attachFrame(page, 'frame1', server.EMPTY_PAGE); - const otherFrame = page.frames()[1]!; - const watchdog = page.waitForSelector('div'); - await otherFrame.evaluate(addElement, 'div'); - await page.evaluate(addElement, 'div'); - const eHandle = await watchdog; - expect(eHandle?.frame).toBe(page.mainFrame()); - } - ); + await page.goto(server.EMPTY_PAGE); + await attachFrame(page, 'frame1', server.EMPTY_PAGE); + const otherFrame = page.frames()[1]!; + const watchdog = page.waitForSelector('div'); + await otherFrame.evaluate(addElement, 'div'); + await page.evaluate(addElement, 'div'); + const eHandle = await watchdog; + expect(eHandle?.frame).toBe(page.mainFrame()); + }); - itFailsFirefox('should run in specified frame', async () => { + it('should run in specified frame', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -495,7 +492,7 @@ describe('waittask specs', function () { expect(eHandle?.frame).toBe(frame2); }); - itFailsFirefox('should throw when frame is detached', async () => { + it('should throw when frame is detached', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -738,7 +735,7 @@ describe('waittask specs', function () { 'waiting for selector `.//div` failed: timeout 10ms exceeded' ); }); - itFailsFirefox('should run in specified frame', async () => { + it('should run in specified frame', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); @@ -751,7 +748,7 @@ describe('waittask specs', function () { const eHandle = await waitForXPathPromise; expect(eHandle?.frame).toBe(frame2); }); - itFailsFirefox('should throw when frame is detached', async () => { + it('should throw when frame is detached', async () => { const {page, server} = getTestState(); await attachFrame(page, 'frame1', server.EMPTY_PAGE); diff --git a/test/src/worker.spec.ts b/test/src/worker.spec.ts index c45febc3d4ce9..6ff8cfb168290 100644 --- a/test/src/worker.spec.ts +++ b/test/src/worker.spec.ts @@ -18,14 +18,14 @@ import expect from 'expect'; import {ConsoleMessage} from '../../lib/cjs/puppeteer/common/ConsoleMessage.js'; import {WebWorker} from '../../lib/cjs/puppeteer/common/WebWorker.js'; import { - describeFailsFirefox, getTestState, setupTestBrowserHooks, setupTestPageAndContextHooks, } from './mocha-utils.js'; import {waitEvent} from './utils.js'; +import {it} from './mocha-utils.js'; -describeFailsFirefox('Workers', function () { +describe('Workers', function () { setupTestBrowserHooks(); setupTestPageAndContextHooks(); it('Page.workers', async () => { diff --git a/test/tsconfig.json b/test/tsconfig.json index 962644f958df4..6842bd8ed42f2 100644 --- a/test/tsconfig.json +++ b/test/tsconfig.json @@ -9,7 +9,8 @@ }, "include": ["src"], "references": [ - {"path": "../tsconfig.lib.json"}, - {"path": "../utils/testserver/tsconfig.json"} + {"path": "../src/tsconfig.cjs.json"}, + {"path": "../utils/testserver/tsconfig.json"}, + {"path": "../utils/mochaRunner/tsconfig.json"} ] } diff --git a/utils/mochaRunner/README.md b/utils/mochaRunner/README.md new file mode 100644 index 0000000000000..efbafbc3b5e12 --- /dev/null +++ b/utils/mochaRunner/README.md @@ -0,0 +1,47 @@ +# Mocha Runner + +Mocha Runner is a test runner on top of mocha. It uses `/test/TestSuites.json` and `/test/TestExpectations.json` files to run mocha tests in multiple configurations and interpret results. + +## Running tests for Mocha Runner itself. + +``` +npm run build:dev && npx c8 node utils/mochaRunner/lib/test.js +``` + +## Running tests using Mocha Runner + +``` +npm run build:dev && npm run test +``` + +By default, the runner runs all test suites applicable to the current platform. +To pick a test suite, provide the `--test-suite` arguments. For example, + +``` +npm run build:dev && npm run test -- --test-suite chrome-headless +``` + +## TestSuites.json + +Define test suites via the `testSuites` attribute. `parameters` can be used in the `TestExpectations.json` to disable tests +based on parameters. The meaning for parameters is defined in `parameterDefinitons` which tell what env object corresponds +to the given parameter. + +## TestExpectations.json + +An expectation looks like this: + +``` + { + "testIdPattern": "[accessibility.spec]", + "platforms": ["darwin", "win32", "linux"], + "parameters": ["firefox"], + "expectations": ["SKIP"] + } +``` + +`testIdPattern` defines a string that will be used to prefix-match tests. `platforms` defines the platforms the expectation is for (`or`-logic). +`parameters` defines the parameters that the test has to match (`and`-logic). `expectations` is the list of test results that are considered to be acceptable. + +Currently, expectations are updated manually. The test runner outputs the suggested changes to the expectation file if the test run does not match +expectations. diff --git a/utils/mochaRunner/src/main.ts b/utils/mochaRunner/src/main.ts new file mode 100644 index 0000000000000..05f3168ce8ff5 --- /dev/null +++ b/utils/mochaRunner/src/main.ts @@ -0,0 +1,204 @@ +import { + TestExpectation, + MochaResults, + zTestSuiteFile, + zPlatform, + TestSuite, + TestSuiteFile, + Platform, +} from './types.js'; + +import path from 'path'; +import fs from 'fs'; +import os from 'os'; +import {spawn} from 'node:child_process'; +import { + extendProcessEnv, + filterByPlatform, + prettyPrintJSON, + readJSON, + filterByParameters, + getExpectationUpdates, + getSkippedTests, +} from './utils.js'; + +function getApplicableTestSuites( + parsedSuitesFile: TestSuiteFile, + platform: Platform +): TestSuite[] { + const testSuiteArgIdx = process.argv.indexOf('--test-suite'); + let applicableSuites: TestSuite[] = []; + + if (testSuiteArgIdx === -1) { + applicableSuites = filterByPlatform(parsedSuitesFile.testSuites, platform); + } else { + const testSuiteId = process.argv[testSuiteArgIdx + 1]; + const testSuite = parsedSuitesFile.testSuites.find(suite => { + return suite.id === testSuiteId; + }); + + if (!testSuite) { + console.error(`Test suite ${testSuiteId} is not defined`); + process.exit(1); + } + + if (!testSuite.platforms.includes(platform)) { + console.warn( + `Test suite ${testSuiteId} is not enabled for your platform. Running it anyway.` + ); + } + + applicableSuites = [testSuite]; + } + + return applicableSuites; +} + +async function main() { + const platform = zPlatform.parse(os.platform()); + + const expectations = readJSON( + path.join(process.cwd(), 'test', 'TestExpectations.json') + ) as TestExpectation[]; + + const parsedSuitesFile = zTestSuiteFile.parse( + readJSON(path.join(process.cwd(), 'test', 'TestSuites.json')) + ); + + const applicableSuites = getApplicableTestSuites(parsedSuitesFile, platform); + + console.log('Planning to run the following test suites', applicableSuites); + + let fail = false; + const recommendations = []; + try { + for (const suite of applicableSuites) { + const parameters = suite.parameters; + + const applicableExpectations = filterByParameters( + filterByPlatform(expectations, platform), + parameters + ); + + const skippedTests = getSkippedTests(applicableExpectations); + + const env = extendProcessEnv([ + ...parameters.map(param => { + return parsedSuitesFile.parameterDefinitons[param]; + }), + { + PUPPETEER_SKIPPED_TEST_CONFIG: JSON.stringify( + skippedTests.map(ex => { + return ex.testIdPattern; + }) + ), + }, + ]); + + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), 'puppeteer-test-runner-') + ); + const tmpFilename = path.join(tmpDir, 'output.json'); + console.log('Running', JSON.stringify(parameters), tmpFilename); + const handle = spawn( + 'npx mocha', + ['--reporter=json', '--reporter-option', 'output=' + tmpFilename], + { + shell: true, + cwd: process.cwd(), + stdio: 'inherit', + env, + } + ); + await new Promise((resolve, reject) => { + handle.on('error', err => { + reject(err); + }); + handle.on('close', () => { + resolve(); + }); + }); + console.log('Finished', JSON.stringify(parameters)); + try { + const results = readJSON(tmpFilename) as MochaResults; + console.log('Results from mocha'); + console.log('Stats', JSON.stringify(results.stats)); + results.pending.length > 0 && console.log('# Pending tests'); + for (const test of results.pending) { + console.log(`\t? ${test.fullTitle} ${test.file}`); + } + results.failures.length > 0 && console.log('# Failed tests'); + for (const test of results.failures) { + console.log(`\tF ${test.fullTitle} ${test.file}`, test.err); + } + const recommendation = getExpectationUpdates( + results, + applicableExpectations, + { + platforms: [os.platform()], + parameters, + } + ); + if (recommendation.length > 0) { + fail = true; + recommendations.push(...recommendation); + } else { + console.log('Test run matches expecations'); + continue; + } + } catch (err) { + fail = true; + console.error(err); + } + } + } catch (err) { + fail = true; + console.error(err); + } finally { + const toAdd = recommendations.filter(item => { + return item.action === 'add'; + }); + if (toAdd.length) { + console.log( + 'Add the following to TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toAdd.map(item => { + return item.expectation; + }) + ); + } + const toRemove = recommendations.filter(item => { + return item.action === 'remove'; + }); + if (toRemove.length) { + console.log( + 'Remove the following from the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toRemove.map(item => { + return item.expectation; + }) + ); + } + const toUpdate = recommendations.filter(item => { + return item.action === 'update'; + }); + if (toUpdate.length) { + console.log( + 'Update the following expectations in the TestExpecations.json to ignore the error:' + ); + prettyPrintJSON( + toUpdate.map(item => { + return item.expectation; + }) + ); + } + process.exit(fail ? 1 : 0); + } +} + +main().catch(error => { + console.error(error); + process.exit(1); +}); diff --git a/utils/mochaRunner/src/test.ts b/utils/mochaRunner/src/test.ts new file mode 100644 index 0000000000000..2de6fbe460722 --- /dev/null +++ b/utils/mochaRunner/src/test.ts @@ -0,0 +1,46 @@ +import assert from 'assert/strict'; +import test from 'node:test'; +import {filterByParameters, getTestResultForFailure} from './utils.js'; +import {TestExpectation} from './types.js'; +import {getFilename, extendProcessEnv} from './utils.js'; + +test('extendProcessEnv', () => { + const env = extendProcessEnv([{TEST: 'TEST'}, {TEST2: 'TEST2'}]); + assert.equal(env['TEST'], 'TEST'); + assert.equal(env['TEST2'], 'TEST2'); +}); + +test('getFilename', () => { + assert.equal(getFilename('/etc/test.ts'), 'test'); + assert.equal(getFilename('/etc/test.js'), 'test'); +}); + +test('getTestResultForFailure', () => { + assert.equal( + getTestResultForFailure({err: {code: 'ERR_MOCHA_TIMEOUT'}}), + 'TIMEOUT' + ); + assert.equal(getTestResultForFailure({err: {code: 'ERROR'}}), 'FAIL'); +}); + +test('filterByParameters', () => { + const expectations: TestExpectation[] = [ + { + testIdPattern: + '[oopif.spec] OOPIF "after all" hook for "should keep track of a frames OOP state"', + platforms: ['darwin'], + parameters: ['firefox', 'headless'], + expectations: ['FAIL'], + }, + ]; + assert.equal( + filterByParameters(expectations, ['firefox', 'headless']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['firefox']).length, 0); + assert.equal( + filterByParameters(expectations, ['firefox', 'headless', 'other']).length, + 1 + ); + assert.equal(filterByParameters(expectations, ['other']).length, 0); +}); diff --git a/utils/mochaRunner/src/types.ts b/utils/mochaRunner/src/types.ts new file mode 100644 index 0000000000000..7fe7b7116866c --- /dev/null +++ b/utils/mochaRunner/src/types.ts @@ -0,0 +1,42 @@ +import {z} from 'zod'; + +export const zPlatform = z.enum(['win32', 'linux', 'darwin']); + +export type Platform = z.infer; + +export const zTestSuite = z.object({ + id: z.string(), + platforms: z.array(zPlatform), + parameters: z.array(z.string()), +}); + +export type TestSuite = z.infer; + +export const zTestSuiteFile = z.object({ + testSuites: z.array(zTestSuite), + parameterDefinitons: z.record(z.any()), +}); + +export type TestSuiteFile = z.infer; + +export type TestResult = 'PASS' | 'FAIL' | 'TIMEOUT' | 'SKIP'; + +export type TestExpectation = { + testIdPattern: string; + platforms: NodeJS.Platform[]; + parameters: string[]; + expectations: TestResult[]; +}; + +export type MochaTestResult = { + fullTitle: string; + file: string; + err?: {code: string}; +}; + +export type MochaResults = { + stats: unknown; + pending: MochaTestResult[]; + passes: MochaTestResult[]; + failures: MochaTestResult[]; +}; diff --git a/utils/mochaRunner/src/utils.ts b/utils/mochaRunner/src/utils.ts new file mode 100644 index 0000000000000..d6deb0efa7a8d --- /dev/null +++ b/utils/mochaRunner/src/utils.ts @@ -0,0 +1,153 @@ +import { + MochaTestResult, + TestExpectation, + MochaResults, + TestResult, +} from './types.js'; +import path from 'path'; +import fs from 'fs'; + +export function extendProcessEnv(envs: object[]): NodeJS.ProcessEnv { + return envs.reduce( + (acc: object, item: object) => { + Object.assign(acc, item); + return acc; + }, + { + ...process.env, + } + ) as NodeJS.ProcessEnv; +} + +export function getFilename(file: string): string { + return path.basename(file).replace(path.extname(file), ''); +} + +export function readJSON(path: string): unknown { + return JSON.parse(fs.readFileSync(path, 'utf-8')); +} + +export function filterByPlatform( + items: T[], + platform: NodeJS.Platform +): T[] { + return items.filter(item => { + return item.platforms.includes(platform); + }); +} + +export function prettyPrintJSON(json: unknown): void { + console.log(JSON.stringify(json, null, 2)); +} + +export function filterByParameters( + expecations: TestExpectation[], + parameters: string[] +): TestExpectation[] { + const querySet = new Set(parameters); + return expecations.filter(ex => { + return ex.parameters.every(param => { + return querySet.has(param); + }); + }); +} + +/** + * The last expectation that matches the startsWith filter wins. + */ +export function findEffectiveExpecationForTest( + expectations: TestExpectation[], + result: MochaTestResult +): TestExpectation | undefined { + return expectations + .filter(expecation => { + if ( + getTestId(result.file, result.fullTitle).startsWith( + expecation.testIdPattern + ) + ) { + return true; + } + return false; + }) + .pop(); +} + +type RecommendedExpecation = { + expectation: TestExpectation; + test: MochaTestResult; + action: 'remove' | 'add' | 'update'; +}; + +export function getExpectationUpdates( + results: MochaResults, + expecations: TestExpectation[], + context: { + platforms: NodeJS.Platform[]; + parameters: string[]; + } +): RecommendedExpecation[] { + const output: RecommendedExpecation[] = []; + + for (const pass of results.passes) { + const expectation = findEffectiveExpecationForTest(expecations, pass); + if (expectation && !expectation.expectations.includes('PASS')) { + output.push({ + expectation, + test: pass, + action: 'remove', + }); + } + } + + for (const failure of results.failures) { + const expectation = findEffectiveExpecationForTest(expecations, failure); + if (expectation) { + if ( + !expectation.expectations.includes(getTestResultForFailure(failure)) + ) { + output.push({ + expectation: { + ...expectation, + expectations: [ + ...expectation.expectations, + getTestResultForFailure(failure), + ], + }, + test: failure, + action: 'update', + }); + } + } else { + output.push({ + expectation: { + testIdPattern: getTestId(failure.file, failure.fullTitle), + platforms: context.platforms, + parameters: context.parameters, + expectations: [getTestResultForFailure(failure)], + }, + test: failure, + action: 'add', + }); + } + } + return output; +} + +export function getTestResultForFailure( + test: Pick +): TestResult { + return test.err?.code === 'ERR_MOCHA_TIMEOUT' ? 'TIMEOUT' : 'FAIL'; +} + +export function getSkippedTests( + expectations: TestExpectation[] +): TestExpectation[] { + return expectations.filter(ex => { + return ex.expectations.includes('SKIP'); + }); +} + +export function getTestId(file: string, fullTitle: string): string { + return `[${getFilename(file)}] ${fullTitle}`; +} diff --git a/utils/mochaRunner/tsconfig.json b/utils/mochaRunner/tsconfig.json new file mode 100644 index 0000000000000..c2576c2564f7f --- /dev/null +++ b/utils/mochaRunner/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "allowJs": true, + "composite": true, + "module": "CommonJS", + "outDir": "lib", + "rootDir": "src" + }, + "include": ["src"] +}