From 907290b2fdc0f774ce636429ca74e14de5754c2c Mon Sep 17 00:00:00 2001 From: Vitalii Tverdokhlib Date: Sat, 11 Sep 2021 13:07:29 +0300 Subject: [PATCH 1/7] chore(docs): update alpine version (#5099) --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 87350bf62d9b3..95b6834aae7a0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -332,7 +332,7 @@ The [newest Chromium package](https://pkgs.alpinelinux.org/package/edge/communit Example Dockerfile: ```Dockerfile -FROM alpine:edge +FROM alpine # Installs latest Chromium (89) package. RUN apk add --no-cache \ From fbd36a9705f64613255ef4dff223e4ddc8576253 Mon Sep 17 00:00:00 2001 From: Ondra Urban <23726914+mnmkng@users.noreply.github.com> Date: Sat, 11 Sep 2021 12:31:59 +0200 Subject: [PATCH 2/7] docs(examples): add Apify SDK to scraping section (#5338) --- examples/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/README.md b/examples/README.md index c23d9df64b5af..ed68e3fe12be5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -23,6 +23,7 @@ More complex and use case driven examples can be found at [github.com/GoogleChro - [puppeteer-examples](https://github.com/checkly/puppeteer-examples) - Puppeteer Headless Chrome examples for real life use cases such as getting useful info from the web pages or common login scenarios. - [browserless](https://github.com/joelgriffith/browserless) - Headless Chrome as a service letting you execute Puppeteer scripts remotely. Provides a docker image with configuration for concurrency, launch arguments and more. - [Puppeteer Sandbox](https://puppeteersandbox.com) - Puppeteer sandbox environment as a service. Runs Puppeteer scripts and allows saving and embedding them in external sites and markdown files. +- [Apify SDK](https://github.com/apifytech/apify-js) - The scalable web crawling and scraping library for JavaScript. Automatically manages a pool of Puppeteer browsers and provides easy error handling, task management, proxy rotation and more. ## Testing From 8d1a697fc3ef10300d886beb961a45ea1451c427 Mon Sep 17 00:00:00 2001 From: Talendran <69163150+Talendran@users.noreply.github.com> Date: Sat, 11 Sep 2021 07:02:40 -0400 Subject: [PATCH 3/7] docs(examples): change searching for searchbox to .devsite class (#6301) --- examples/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/search.js b/examples/search.js index 828d15543815a..b5def52f733e7 100644 --- a/examples/search.js +++ b/examples/search.js @@ -30,7 +30,7 @@ const puppeteer = require('puppeteer'); await page.goto('https://developers.google.com/web/'); // Type into search box. - await page.type('.devsite-searchbox input', 'Headless Chrome'); + await page.type('.devsite-search-field', 'Headless Chrome'); // Wait for suggest overlay to appear and click "show all results". const allResultsSelector = '.devsite-suggest-all-results'; From b5020dc04121b265c77662237dfb177d6de06053 Mon Sep 17 00:00:00 2001 From: Michael Rienstra Date: Sat, 11 Sep 2021 10:50:35 -0700 Subject: [PATCH 4/7] feat: add more Android models to DeviceDescriptors (#7210) Adds device descriptions for: * Galaxy S8 * Galaxy S9+ * Galaxy Tab S4 * Pixel 3 * Pixel 4 These devices are regarded as worthy of targeting by [BrowserStack](https://www.browserstack.com/test-on-the-right-mobile-devices). Sources (both have identical data for these 5 devices): 1. https://github.com/aerokube/moon-deploy/blob/master/moon-local.yaml#L199 2. https://www.danhendricks.com/2018/04/adding-iphone-galaxy-chrome-mobile-emulated-devices/#heading_device_data --- src/common/DeviceDescriptors.ts | 130 ++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/src/common/DeviceDescriptors.ts b/src/common/DeviceDescriptors.ts index 783a947ae3592..7b5b54c96837d 100644 --- a/src/common/DeviceDescriptors.ts +++ b/src/common/DeviceDescriptors.ts @@ -187,6 +187,84 @@ const devices: Device[] = [ isLandscape: true, }, }, + { + name: 'Galaxy S8', + userAgent: + 'Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36', + viewport: { + width: 360, + height: 740, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, + isLandscape: false, + }, + }, + { + name: 'Galaxy S8 landscape', + userAgent: + 'Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36', + viewport: { + width: 740, + height: 360, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, + isLandscape: true, + }, + }, + { + name: 'Galaxy S9+', + userAgent: + 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36', + viewport: { + width: 320, + height: 658, + deviceScaleFactor: 4.5, + isMobile: true, + hasTouch: true, + isLandscape: false, + }, + }, + { + name: 'Galaxy S9+ landscape', + userAgent: + 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36', + viewport: { + width: 658, + height: 320, + deviceScaleFactor: 4.5, + isMobile: true, + hasTouch: true, + isLandscape: true, + }, + }, + { + name: 'Galaxy Tab S4', + userAgent: + 'Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36', + viewport: { + width: 712, + height: 1138, + deviceScaleFactor: 2.25, + isMobile: true, + hasTouch: true, + isLandscape: false, + }, + }, + { + name: 'Galaxy Tab S4 landscape', + userAgent: + 'Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36', + viewport: { + width: 1138, + height: 712, + deviceScaleFactor: 2.25, + isMobile: true, + hasTouch: true, + isLandscape: true, + }, + }, { name: 'iPad', userAgent: @@ -1032,6 +1110,58 @@ const devices: Device[] = [ isLandscape: true, }, }, + { + name: 'Pixel 3', + userAgent: + 'Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36', + viewport: { + width: 393, + height: 786, + deviceScaleFactor: 2.75, + isMobile: true, + hasTouch: true, + isLandscape: false, + }, + }, + { + name: 'Pixel 3 landscape', + userAgent: + 'Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36', + viewport: { + width: 786, + height: 393, + deviceScaleFactor: 2.75, + isMobile: true, + hasTouch: true, + isLandscape: true, + }, + }, + { + name: 'Pixel 4', + userAgent: + 'Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36', + viewport: { + width: 353, + height: 745, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, + isLandscape: false, + }, + }, + { + name: 'Pixel 4 landscape', + userAgent: + 'Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36', + viewport: { + width: 745, + height: 353, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, + isLandscape: true, + }, + }, ]; /** * @public From 3c6029c702291ca7ef637b66e78d72e03156fe58 Mon Sep 17 00:00:00 2001 From: Tom Jenkinson Date: Sat, 11 Sep 2021 21:28:12 +0100 Subject: [PATCH 5/7] feat(api): implement `Page.waitForNetworkIdle()` (#5140) which will wait for there to be no network requests in progress during the `idleTime` before resolving. --- docs/api.md | 12 ++++++ src/common/NetworkManager.ts | 6 +++ src/common/Page.ts | 73 ++++++++++++++++++++++++++++++++++++ test/page.spec.ts | 73 ++++++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+) diff --git a/docs/api.md b/docs/api.md index ab1c2c6ec0d1e..6d73a1c2457be 100644 --- a/docs/api.md +++ b/docs/api.md @@ -197,6 +197,7 @@ * [page.waitForFileChooser([options])](#pagewaitforfilechooseroptions) * [page.waitForFunction(pageFunction[, options[, ...args]])](#pagewaitforfunctionpagefunction-options-args) * [page.waitForNavigation([options])](#pagewaitfornavigationoptions) + * [page.waitForNetworkIdle([options])](#pagewaitfornetworkidleoptions) * [page.waitForRequest(urlOrPredicate[, options])](#pagewaitforrequesturlorpredicate-options) * [page.waitForResponse(urlOrPredicate[, options])](#pagewaitforresponseurlorpredicate-options) * [page.waitForSelector(selector[, options])](#pagewaitforselectorselector-options) @@ -2846,6 +2847,17 @@ const [response] = await Promise.all([ Shortcut for [page.mainFrame().waitForNavigation(options)](#framewaitfornavigationoptions). +#### page.waitForNetworkIdle([options]) +- `options` <[Object]> Optional waiting parameters + - `timeout` <[number]> Maximum wait time in milliseconds, defaults to 30 seconds, pass `0` to disable the timeout. The default value can be changed by using the [page.setDefaultTimeout(timeout)](#pagesetdefaulttimeouttimeout) method. + - `idleTime` <[number]> How long to wait for no network requests in milliseconds, defaults to 500 milliseconds. +- returns: <[Promise]> Promise which resolves when network is idle. + +```js +page.evaluate(() => fetch('some-url')); +page.waitForNetworkIdle(); // The promise resolves after fetch above finishes +``` + #### page.waitForRequest(urlOrPredicate[, options]) - `urlOrPredicate` <[string]|[Function]> A URL or predicate to wait for. diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index dda138e693fd9..73ae84a9d512e 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -188,6 +188,12 @@ export class NetworkManager extends EventEmitter { return Object.assign({}, this._extraHTTPHeaders); } + numRequestsInProgress(): number { + return [...this._requestIdToRequest].filter(([, request]) => { + return !request.response(); + }).length; + } + async setOfflineMode(value: boolean): Promise { this._emulatedNetworkConditions.offline = value; await this._updateNetworkConditions(); diff --git a/src/common/Page.ts b/src/common/Page.ts index c5caaaf57bacf..b2aa354d9eb30 100644 --- a/src/common/Page.ts +++ b/src/common/Page.ts @@ -1894,6 +1894,79 @@ export class Page extends EventEmitter { ); } + /** + * @param options - Optional waiting parameters + * @returns Promise which resolves when network is idle + */ + async waitForNetworkIdle( + options: { idleTime?: number; timeout?: number } = {} + ): Promise { + const { idleTime = 500, timeout = this._timeoutSettings.timeout() } = + options; + + const networkManager = this._frameManager.networkManager(); + + let idleResolveCallback; + const idlePromise = new Promise((resolve) => { + idleResolveCallback = resolve; + }); + + let abortRejectCallback; + const abortPromise = new Promise((_, reject) => { + abortRejectCallback = reject; + }); + + let idleTimer; + const onIdle = () => idleResolveCallback(); + + const cleanup = () => { + idleTimer && clearTimeout(idleTimer); + abortRejectCallback(new Error('abort')); + }; + + const evaluate = () => { + idleTimer && clearTimeout(idleTimer); + if (networkManager.numRequestsInProgress() === 0) + idleTimer = setTimeout(onIdle, idleTime); + }; + + evaluate(); + + const eventHandler = () => { + evaluate(); + return false; + }; + + const listenToEvent = (event) => + helper.waitForEvent( + networkManager, + event, + eventHandler, + timeout, + abortPromise + ); + + const eventPromises = [ + listenToEvent(NetworkManagerEmittedEvents.Request), + listenToEvent(NetworkManagerEmittedEvents.Response), + ]; + + await Promise.race([ + idlePromise, + ...eventPromises, + this._sessionClosePromise(), + ]).then( + (r) => { + cleanup(); + return r; + }, + (error) => { + cleanup(); + throw error; + } + ); + } + /** * This method navigate to the previous page in history. * @param options - Navigation parameters diff --git a/test/page.spec.ts b/test/page.spec.ts index f81851c5fad4b..5c9ef3a09af1d 100644 --- a/test/page.spec.ts +++ b/test/page.spec.ts @@ -825,6 +825,79 @@ describe('Page', function () { }); }); + describe('Page.waitForNetworkIdle', function () { + it('should work', async () => { + const { page, server } = getTestState(); + await page.goto(server.EMPTY_PAGE); + let res; + const [t1, t2] = await Promise.all([ + page.waitForNetworkIdle().then((r) => { + res = r; + return Date.now(); + }), + page + .evaluate(() => + (async () => { + await Promise.all([ + fetch('/digits/1.png'), + fetch('/digits/2.png'), + ]); + await new Promise((resolve) => setTimeout(resolve, 200)); + await fetch('/digits/3.png'); + await new Promise((resolve) => setTimeout(resolve, 400)); + await fetch('/digits/4.png'); + })() + ) + .then(() => Date.now()), + ]); + expect(res).toBe(undefined); + expect(t1).toBeGreaterThan(t2); + expect(t1 - t2).toBeGreaterThanOrEqual(400); + }); + it('should respect timeout', async () => { + const { page, puppeteer } = getTestState(); + let error = null; + await page + .waitForNetworkIdle({ timeout: 1 }) + .catch((error_) => (error = error_)); + expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError); + }); + it('should respect idleTime', async () => { + const { page, server } = getTestState(); + await page.goto(server.EMPTY_PAGE); + const [t1, t2] = await Promise.all([ + page.waitForNetworkIdle({ idleTime: 10 }).then(() => Date.now()), + page + .evaluate(() => + (async () => { + await Promise.all([ + fetch('/digits/1.png'), + fetch('/digits/2.png'), + ]); + await new Promise((resolve) => setTimeout(resolve, 250)); + })() + ) + .then(() => Date.now()), + ]); + expect(t2).toBeGreaterThan(t1); + }); + it('should work with no timeout', async () => { + const { page, server } = getTestState(); + await page.goto(server.EMPTY_PAGE); + const [result] = await Promise.all([ + page.waitForNetworkIdle({ timeout: 0 }), + page.evaluate(() => + setTimeout(() => { + fetch('/digits/1.png'); + fetch('/digits/2.png'); + fetch('/digits/3.png'); + }, 50) + ), + ]); + expect(result).toBe(undefined); + }); + }); + describeFailsFirefox('Page.exposeFunction', function () { it('should work', async () => { const { page } = getTestState(); From 723052d5bb3c3d1d3908508467512bea4d8fdc80 Mon Sep 17 00:00:00 2001 From: Remco Haszing Date: Sat, 11 Sep 2021 22:59:51 +0200 Subject: [PATCH 6/7] feat(typescript): allow using puppeteer without dom lib (#6998) The dom lib inserts all dom related types into the project, which is often undesirable when working on a NodeJS project. This change injects global stubs for the dom types required by puppeteer, so puppeteer can work without users having to add dom types to their project. Closes #6989 --- inject-global-type-stubs.js | 29 +++++++++++++++++++++++++++++ package.json | 2 +- src/global.ts | 20 ++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 inject-global-type-stubs.js create mode 100644 src/global.ts diff --git a/inject-global-type-stubs.js b/inject-global-type-stubs.js new file mode 100644 index 0000000000000..adc634990321c --- /dev/null +++ b/inject-global-type-stubs.js @@ -0,0 +1,29 @@ +/** + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This script is needed because of https://github.com/microsoft/rushstack/issues/1709 +const { promises: fs } = require('fs'); +const { join } = require('path'); + +async function injctGlobalTypeStubs() { + const typesPath = join(__dirname, 'lib', 'types.d.ts'); + const globalsPath = join(__dirname, 'lib', 'cjs', 'puppeteer', 'global.d.ts'); + const types = await fs.readFile(typesPath, 'utf-8'); + const globals = await fs.readFile(globalsPath, 'utf-8'); + await fs.writeFile(typesPath, `${globals}\n${types}`); +} + +injctGlobalTypeStubs(); diff --git a/package.json b/package.json index e960d7ab34cdd..61ae8dc256d05 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "apply-next-version": "node utils/apply_next_version.js", "test-install": "scripts/test-install.sh", "clean-docs": "rimraf website/docs && rimraf docs-api-json", - "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose", + "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose && node inject-global-type-stubs.js", "generate-docs": "npm run generate-d-ts && api-documenter markdown -i docs-api-json -o website/docs && node utils/remove-tag.js", "ensure-correct-devtools-protocol-revision": "ts-node -s scripts/ensure-correct-devtools-protocol-package", "ensure-pinned-deps": "ts-node -s scripts/ensure-pinned-deps", diff --git a/src/global.ts b/src/global.ts new file mode 100644 index 0000000000000..2dce058df26ee --- /dev/null +++ b/src/global.ts @@ -0,0 +1,20 @@ +/** + * These global declarations exist so puppeteer can work without the need to use `"dom"` + * types. + * + * To get full type information for these interfaces, add `"types": "dom"`in your + * `tsconfig.json` file. + */ +declare global { + // eslint-disable-next-line @typescript-eslint/no-empty-interface + interface Document {} + + // eslint-disable-next-line @typescript-eslint/no-empty-interface + interface Element {} + + // eslint-disable-next-line max-len + // eslint-disable-next-line @typescript-eslint/no-empty-interface, @typescript-eslint/no-unused-vars + interface NodeListOf {} +} + +export {}; From 491614c7f8cfa50b902d0275064e611c2a48c3b2 Mon Sep 17 00:00:00 2001 From: Albert Nigmatzianov Date: Sat, 11 Sep 2021 23:31:08 +0200 Subject: [PATCH 7/7] fix(types): allow evaluate functions to take a readonly array as an argument (#7072) --- src/common/EvalTypes.ts | 2 +- test/frame.spec.ts | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/common/EvalTypes.ts b/src/common/EvalTypes.ts index 200cf70129ab3..b400f44b5fb47 100644 --- a/src/common/EvalTypes.ts +++ b/src/common/EvalTypes.ts @@ -54,7 +54,7 @@ export type Serializable = /** * @public */ -export type JSONArray = Serializable[]; +export type JSONArray = readonly Serializable[]; /** * @public diff --git a/test/frame.spec.ts b/test/frame.spec.ts index 269da7d7f1be8..f1310f6905dfd 100644 --- a/test/frame.spec.ts +++ b/test/frame.spec.ts @@ -79,6 +79,17 @@ describe('Frame specs', function () { 'Execution context is not available in detached frame' ); }); + + it('allows readonly array to be an argument', async () => { + const { page, server } = getTestState(); + await page.goto(server.EMPTY_PAGE); + const mainFrame = page.mainFrame(); + + // This test checks if Frame.evaluate allows a readonly array to be an argument. + // See https://github.com/puppeteer/puppeteer/issues/6953. + const readonlyArray: readonly string[] = ['a', 'b', 'c']; + mainFrame.evaluate((arr) => arr, readonlyArray); + }); }); describe('Frame Management', function () {