Merge remote-tracking branch 'origin/main'

docs/api.md

@@ -197,6 +197,7 @@
   * [page.waitForFileChooser([options])](#pagewaitforfilechooseroptions)
   * [page.waitForFunction(pageFunction[, options[, ...args]])](#pagewaitforfunctionpagefunction-options-args)
   * [page.waitForNavigation([options])](#pagewaitfornavigationoptions)
+  * [page.waitForNetworkIdle([options])](#pagewaitfornetworkidleoptions)
   * [page.waitForRequest(urlOrPredicate[, options])](#pagewaitforrequesturlorpredicate-options)
   * [page.waitForResponse(urlOrPredicate[, options])](#pagewaitforresponseurlorpredicate-options)
   * [page.waitForSelector(selector[, options])](#pagewaitforselectorselector-options)
@@ -2847,6 +2848,17 @@ const [response] = await Promise.all([
 
 Shortcut for [page.mainFrame().waitForNavigation(options)](#framewaitfornavigationoptions).
 
+#### page.waitForNetworkIdle([options])
+- `options` <[Object]> Optional waiting parameters
+  - `timeout` <[number]> Maximum wait time in milliseconds, defaults to 30 seconds, pass `0` to disable the timeout. The default value can be changed by using the [page.setDefaultTimeout(timeout)](#pagesetdefaulttimeouttimeout) method.
+  - `idleTime` <[number]> How long to wait for no network requests in milliseconds, defaults to 500 milliseconds.
+- returns: <[Promise]<void>> Promise which resolves when network is idle.
+
+```js
+page.evaluate(() => fetch('some-url'));
+page.waitForNetworkIdle(); // The promise resolves after fetch above finishes
+```
+
 #### page.waitForRequest(urlOrPredicate[, options])
 
 - `urlOrPredicate` <[string]|[Function]> A URL or predicate to wait for.

docs/troubleshooting.md

@@ -332,7 +332,7 @@ The [newest Chromium package](https://pkgs.alpinelinux.org/package/edge/communit
 Example Dockerfile:
 
 ```Dockerfile
-FROM alpine:edge
+FROM alpine
 
 # Installs latest Chromium (89) package.
 RUN apk add --no-cache \

examples/README.md

@@ -23,6 +23,7 @@ More complex and use case driven examples can be found at [github.com/GoogleChro
 - [puppeteer-examples](https://github.com/checkly/puppeteer-examples) - Puppeteer Headless Chrome examples for real life use cases such as getting useful info from the web pages or common login scenarios.
 - [browserless](https://github.com/joelgriffith/browserless) - Headless Chrome as a service letting you execute Puppeteer scripts remotely. Provides a docker image with configuration for concurrency, launch arguments and more.
 - [Puppeteer Sandbox](https://puppeteersandbox.com) - Puppeteer sandbox environment as a service. Runs Puppeteer scripts and allows saving and embedding them in external sites and markdown files.
+- [Apify SDK](https://github.com/apifytech/apify-js) - The scalable web crawling and scraping library for JavaScript. Automatically manages a pool of Puppeteer browsers and provides easy error handling, task management, proxy rotation and more.
 
 ## Testing
 

examples/search.js

@@ -30,7 +30,7 @@ const puppeteer = require('puppeteer');
   await page.goto('https://developers.google.com/web/');
 
   // Type into search box.
-  await page.type('.devsite-searchbox input', 'Headless Chrome');
+  await page.type('.devsite-search-field', 'Headless Chrome');
 
   // Wait for suggest overlay to appear and click "show all results".
   const allResultsSelector = '.devsite-suggest-all-results';

inject-global-type-stubs.js

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2021 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This script is needed because of https://github.com/microsoft/rushstack/issues/1709
+const { promises: fs } = require('fs');
+const { join } = require('path');
+
+async function injctGlobalTypeStubs() {
+  const typesPath = join(__dirname, 'lib', 'types.d.ts');
+  const globalsPath = join(__dirname, 'lib', 'cjs', 'puppeteer', 'global.d.ts');
+  const types = await fs.readFile(typesPath, 'utf-8');
+  const globals = await fs.readFile(globalsPath, 'utf-8');
+  await fs.writeFile(typesPath, `${globals}\n${types}`);
+}
+
+injctGlobalTypeStubs();

package.json

@@ -36,7 +36,7 @@
     "apply-next-version": "node utils/apply_next_version.js",
     "test-install": "scripts/test-install.sh",
     "clean-docs": "rimraf website/docs && rimraf docs-api-json",
-    "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose",
+    "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose && node inject-global-type-stubs.js",
     "generate-docs": "npm run generate-d-ts && api-documenter markdown -i docs-api-json -o website/docs && node utils/remove-tag.js",
     "ensure-correct-devtools-protocol-revision": "ts-node -s scripts/ensure-correct-devtools-protocol-package",
     "ensure-pinned-deps": "ts-node -s scripts/ensure-pinned-deps",

src/common/DeviceDescriptors.ts

@@ -187,6 +187,84 @@ const devices: Device[] = [
       isLandscape: true,
     },
   },
+  {
+    name: 'Galaxy S8',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36',
+    viewport: {
+      width: 360,
+      height: 740,
+      deviceScaleFactor: 3,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: false,
+    },
+  },
+  {
+    name: 'Galaxy S8 landscape',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 7.0; SM-G950U Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36',
+    viewport: {
+      width: 740,
+      height: 360,
+      deviceScaleFactor: 3,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: true,
+    },
+  },
+  {
+    name: 'Galaxy S9+',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36',
+    viewport: {
+      width: 320,
+      height: 658,
+      deviceScaleFactor: 4.5,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: false,
+    },
+  },
+  {
+    name: 'Galaxy S9+ landscape',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 8.0.0; SM-G965U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.111 Mobile Safari/537.36',
+    viewport: {
+      width: 658,
+      height: 320,
+      deviceScaleFactor: 4.5,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: true,
+    },
+  },
+  {
+    name: 'Galaxy Tab S4',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36',
+    viewport: {
+      width: 712,
+      height: 1138,
+      deviceScaleFactor: 2.25,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: false,
+    },
+  },
+  {
+    name: 'Galaxy Tab S4 landscape',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 8.1.0; SM-T837A) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Safari/537.36',
+    viewport: {
+      width: 1138,
+      height: 712,
+      deviceScaleFactor: 2.25,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: true,
+    },
+  },
   {
     name: 'iPad',
     userAgent:
@@ -1032,6 +1110,58 @@ const devices: Device[] = [
       isLandscape: true,
     },
   },
+  {
+    name: 'Pixel 3',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36',
+    viewport: {
+      width: 393,
+      height: 786,
+      deviceScaleFactor: 2.75,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: false,
+    },
+  },
+  {
+    name: 'Pixel 3 landscape',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 9; Pixel 3 Build/PQ1A.181105.017.A1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.158 Mobile Safari/537.36',
+    viewport: {
+      width: 786,
+      height: 393,
+      deviceScaleFactor: 2.75,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: true,
+    },
+  },
+  {
+    name: 'Pixel 4',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36',
+    viewport: {
+      width: 353,
+      height: 745,
+      deviceScaleFactor: 3,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: false,
+    },
+  },
+  {
+    name: 'Pixel 4 landscape',
+    userAgent:
+      'Mozilla/5.0 (Linux; Android 10; Pixel 4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Mobile Safari/537.36',
+    viewport: {
+      width: 745,
+      height: 353,
+      deviceScaleFactor: 3,
+      isMobile: true,
+      hasTouch: true,
+      isLandscape: true,
+    },
+  },
 ];
 /**
  * @public

src/common/EvalTypes.ts

@@ -54,7 +54,7 @@ export type Serializable =
 /**
  * @public
  */
-export type JSONArray = Serializable[];
+export type JSONArray = readonly Serializable[];
 
 /**
  * @public

src/common/NetworkManager.ts

@@ -188,6 +188,12 @@ export class NetworkManager extends EventEmitter {
     return Object.assign({}, this._extraHTTPHeaders);
   }
 
+  numRequestsInProgress(): number {
+    return [...this._requestIdToRequest].filter(([, request]) => {
+      return !request.response();
+    }).length;
+  }
+
   async setOfflineMode(value: boolean): Promise<void> {
     this._emulatedNetworkConditions.offline = value;
     await this._updateNetworkConditions();

src/common/Page.ts

@@ -1894,6 +1894,79 @@ export class Page extends EventEmitter {
     );
   }
 
+  /**
+   * @param options - Optional waiting parameters
+   * @returns Promise which resolves when network is idle
+   */
+  async waitForNetworkIdle(
+    options: { idleTime?: number; timeout?: number } = {}
+  ): Promise<void> {
+    const { idleTime = 500, timeout = this._timeoutSettings.timeout() } =
+      options;
+
+    const networkManager = this._frameManager.networkManager();
+
+    let idleResolveCallback;
+    const idlePromise = new Promise((resolve) => {
+      idleResolveCallback = resolve;
+    });
+
+    let abortRejectCallback;
+    const abortPromise = new Promise<Error>((_, reject) => {
+      abortRejectCallback = reject;
+    });
+
+    let idleTimer;
+    const onIdle = () => idleResolveCallback();
+
+    const cleanup = () => {
+      idleTimer && clearTimeout(idleTimer);
+      abortRejectCallback(new Error('abort'));
+    };
+
+    const evaluate = () => {
+      idleTimer && clearTimeout(idleTimer);
+      if (networkManager.numRequestsInProgress() === 0)
+        idleTimer = setTimeout(onIdle, idleTime);
+    };
+
+    evaluate();
+
+    const eventHandler = () => {
+      evaluate();
+      return false;
+    };
+
+    const listenToEvent = (event) =>
+      helper.waitForEvent(
+        networkManager,
+        event,
+        eventHandler,
+        timeout,
+        abortPromise
+      );
+
+    const eventPromises = [
+      listenToEvent(NetworkManagerEmittedEvents.Request),
+      listenToEvent(NetworkManagerEmittedEvents.Response),
+    ];
+
+    await Promise.race([
+      idlePromise,
+      ...eventPromises,
+      this._sessionClosePromise(),
+    ]).then(
+      (r) => {
+        cleanup();
+        return r;
+      },
+      (error) => {
+        cleanup();
+        throw error;
+      }
+    );
+  }
+
   /**
    * This method navigate to the previous page in history.
    * @param options - Navigation parameters

src/global.ts

@@ -0,0 +1,20 @@
+/**
+ * These global declarations exist so puppeteer can work without the need to use `"dom"`
+ * types.
+ *
+ * To get full type information for these interfaces, add `"types": "dom"`in your
+ * `tsconfig.json` file.
+ */
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-empty-interface
+  interface Document {}
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-interface
+  interface Element {}
+
+  // eslint-disable-next-line max-len
+  // eslint-disable-next-line @typescript-eslint/no-empty-interface, @typescript-eslint/no-unused-vars
+  interface NodeListOf<TNode> {}
+}
+
+export {};

test/frame.spec.ts

@@ -79,6 +79,17 @@ describe('Frame specs', function () {
         'Execution context is not available in detached frame'
       );
     });
+
+    it('allows readonly array to be an argument', async () => {
+      const { page, server } = getTestState();
+      await page.goto(server.EMPTY_PAGE);
+      const mainFrame = page.mainFrame();
+
+      // This test checks if Frame.evaluate allows a readonly array to be an argument.
+      // See https://github.com/puppeteer/puppeteer/issues/6953.
+      const readonlyArray: readonly string[] = ['a', 'b', 'c'];
+      mainFrame.evaluate((arr) => arr, readonlyArray);
+    });
   });
 
   describe('Frame Management', function () {