feat: add support for string-based custom queries

CONTRIBUTING.md

@@ -4,6 +4,7 @@
   * [Getting Code](#getting-code)
   * [Code reviews](#code-reviews)
   * [Code Style](#code-style)
+  * [TypeScript guidelines](#typescript-guidelines)
   * [API guidelines](#api-guidelines)
   * [Commit Messages](#commit-messages)
   * [Writing Documentation](#writing-documentation)
@@ -63,16 +64,27 @@ information on using pull requests.
 
 ## Code Style
 
-- Coding style is fully defined in [.eslintrc](https://github.com/puppeteer/puppeteer/blob/master/.eslintrc.js)
-- Code should be annotated with [closure annotations](https://github.com/google/closure-compiler/wiki/Annotating-JavaScript-for-the-Closure-Compiler).
-- Comments should be generally avoided. If the code would not be understood without comments, consider re-writing the code to make it self-explanatory.
+- Coding style is fully defined in [`.eslintrc`](https://github.com/puppeteer/puppeteer/blob/master/.eslintrc.js).
+- We are migrating the Puppeteer codebase to TypeScript which is why you'll find both JS and TS files in `src/`.
+- If you're working in a JS file, code should be annotated with [closure annotations](https://github.com/google/closure-compiler/wiki/Annotating-JavaScript-for-the-Closure-Compiler).
+- If you're working in a TS file, you should explicitly type all variables and return types. You'll get ESLint warnings if you don't so if you're not sure use them as guidelines, and feel free to ask us for help!
 
-To run code linter, use:
+To run ESLint, use:
 
 ```bash
-npm run lint
+npm run eslint
 ```
 
+You can check your code (both JS & TS) type-checks by running:
+
+```bash
+npm run tsc
+```
+
+## TypeScript guidelines
+
+- Try to avoid the use of `any` when possible. Consider `unknown` as a better alternative. You are able to use `any` if needbe, but it will generate an ESLint warning.
+
 ## API guidelines
 
 When authoring new API methods, consider the following:
@@ -145,7 +157,7 @@ A barrier for introducing new installation dependencies is especially high:
 
 - Every feature should be accompanied by a test.
 - Every public api event/method should be accompanied by a test.
-- Tests should be *hermetic*. Tests should not depend on external services.
+- Tests should not depend on external services.
 - Tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests.
 
 Puppeteer tests are located in the test directory ([`test`](https://github.com/puppeteer/puppeteer/blob/master/test/) and are written using Mocha. See [`test/README.md`](https://github.com/puppeteer/puppeteer/blob/master/test/) for more details.
@@ -158,19 +170,6 @@ Despite being named 'unit', these are integration tests, making sure public API
 npm run unit
 ```
 
-- To run tests in parallel, use `-j` flag:
-
-```bash
-npm run unit -- -j 4
-```
-
-- To run tests in "verbose" mode or to stop testrunner on first failure:
-
-```bash
-npm run unit -- --verbose
-npm run unit -- --break-on-failure
-```
-
 - To run a specific test, substitute the `it` with `it.only`:
 
 ```js
@@ -198,6 +197,13 @@ npm run unit -- --break-on-failure
 HEADLESS=false npm run unit
 ```
 
+- To run Firefox tests, firstly ensure you have Firefox installed locally (you only need to do this once, not on every test run) and then you can run the tests:
+
+```bash
+PUPPETEER_PRODUCT=firefox node install.js
+PUPPETEER_PRODUCT=firefox npm run unit
+```
+
 - To run tests with custom browser executable:
 
 ```bash

docs/api.md

@@ -1321,7 +1321,7 @@ const iPhone = puppeteer.devices['iPhone 6'];
 })();
 ```
 
-List of all available devices is available in the source code: [DeviceDescriptors.js](https://github.com/puppeteer/puppeteer/blob/master/lib/DeviceDescriptors.js).
+List of all available devices is available in the source code: [src/DeviceDescriptors.ts](https://github.com/puppeteer/puppeteer/blob/master/src/DeviceDescriptors.ts).
 
 #### page.emulateMedia(type)
 - `type` <?[string]> Changes the CSS media type of the page. The only allowed values are `'screen'`, `'print'` and `null`. Passing `null` disables CSS media emulation.

mocha-config/puppeteer-unit-tests.js

@@ -20,5 +20,7 @@ module.exports = {
   ...base,
   file: ['./test/mocha-utils.js'],
   spec: 'test/*.spec.js',
+  // retry twice more, so we run each test up to 3 times if needed.
+  retries: 2,
   timeout: 25 * 1000,
 };

src/Browser.ts

@@ -23,12 +23,6 @@ import {Connection} from './Connection';
 
 type BrowserCloseCallback = () => Promise<void> | void;
 
-/* TODO(jacktfranklin): once Target is migrated to TS
- * we can import + use its type here. But right now Target
- * is implemented in JS so we can't import the type and have to use
- * Puppeteer.Target
- */
-
 export class Browser extends EventEmitter {
   static async create(connection: Connection, contextIds: string[], ignoreHTTPSErrors: boolean, defaultViewport?: Puppeteer.Viewport, process?: Puppeteer.ChildProcess, closeCallback?: BrowserCloseCallback): Promise<Browser> {
     const browser = new Browser(connection, contextIds, ignoreHTTPSErrors, defaultViewport, process, closeCallback);
@@ -44,7 +38,7 @@ export class Browser extends EventEmitter {
    _defaultContext: BrowserContext;
    _contexts: Map<string, BrowserContext>;
    // TODO: once Target is in TypeScript we can type this properly.
-   _targets: Map<string, Puppeteer.Target>;
+   _targets: Map<string, Target>;
 
    constructor(connection: Connection, contextIds: string[], ignoreHTTPSErrors: boolean, defaultViewport?: Puppeteer.Viewport, process?: Puppeteer.ChildProcess, closeCallback?: BrowserCloseCallback) {
      super();
@@ -154,11 +148,11 @@ export class Browser extends EventEmitter {
      return page;
    }
 
-   targets(): Puppeteer.Target[] {
+   targets(): Target[] {
      return Array.from(this._targets.values()).filter(target => target._isInitialized);
    }
 
-   target(): Puppeteer.Target {
+   target(): Target {
      return this.targets().find(target => target.type() === 'browser');
    }
 
@@ -167,27 +161,27 @@ export class Browser extends EventEmitter {
    * @param {{timeout?: number}=} options
    * @return {!Promise<!Target>}
    */
-   async waitForTarget(predicate: (x: Puppeteer.Target) => boolean, options: { timeout?: number} = {}): Promise<Puppeteer.Target> {
+   async waitForTarget(predicate: (x: Target) => boolean, options: { timeout?: number} = {}): Promise<Target> {
      const {
        timeout = 30000
      } = options;
      const existingTarget = this.targets().find(predicate);
      if (existingTarget)
        return existingTarget;
      let resolve;
-     const targetPromise = new Promise<Puppeteer.Target>(x => resolve = x);
+     const targetPromise = new Promise<Target>(x => resolve = x);
      this.on(Events.Browser.TargetCreated, check);
      this.on(Events.Browser.TargetChanged, check);
      try {
        if (!timeout)
          return await targetPromise;
-       return await helper.waitWithTimeout<Puppeteer.Target>(targetPromise, 'target', timeout);
+       return await helper.waitWithTimeout<Target>(targetPromise, 'target', timeout);
      } finally {
        this.removeListener(Events.Browser.TargetCreated, check);
        this.removeListener(Events.Browser.TargetChanged, check);
      }
 
-     function check(target: Puppeteer.Target): void  {
+     function check(target: Target): void  {
        if (predicate(target))
          resolve(target);
      }
@@ -242,11 +236,11 @@ export class BrowserContext extends EventEmitter {
     this._id = contextId;
   }
 
-  targets(): Puppeteer.Target[] {
+  targets(): Target[] {
     return this._browser.targets().filter(target => target.browserContext() === this);
   }
 
-  waitForTarget(predicate: (x: Puppeteer.Target) => boolean, options: { timeout?: number}): Promise<Puppeteer.Target> {
+  waitForTarget(predicate: (x: Target) => boolean, options: { timeout?: number}): Promise<Target> {
     return this._browser.waitForTarget(target => target.browserContext() === this && predicate(target), options);
   }
 

src/DOMWorld.ts

@@ -23,6 +23,11 @@ import {ExecutionContext} from './ExecutionContext';
 import {TimeoutSettings} from './TimeoutSettings';
 import {MouseButtonInput} from './Input';
 import {FrameManager, Frame} from './FrameManager';
+import {getQueryHandlerAndSelector, QueryHandler} from './QueryHandler';
+
+// This predicateQueryHandler is declared here so that TypeScript knows about it
+// when it is used in the predicate function below.
+declare const predicateQueryHandler: QueryHandler;
 
 const readFileAsync = helper.promisify(fs.readFile);
 
@@ -364,7 +369,7 @@ export class DOMWorld {
       polling = 'raf',
       timeout = this._timeoutSettings.timeout(),
     } = options;
-    return new WaitTask(this, pageFunction, 'function', polling, timeout, ...args).promise;
+    return new WaitTask(this, pageFunction, undefined, 'function', polling, timeout, ...args).promise;
   }
 
   async title(): Promise<string> {
@@ -379,7 +384,8 @@ export class DOMWorld {
     } = options;
     const polling = waitForVisible || waitForHidden ? 'raf' : 'mutation';
     const title = `${isXPath ? 'XPath' : 'selector'} "${selectorOrXPath}"${waitForHidden ? ' to be hidden' : ''}`;
-    const waitTask = new WaitTask(this, predicate, title, polling, timeout, selectorOrXPath, isXPath, waitForVisible, waitForHidden);
+    const {updatedSelector, queryHandler} = getQueryHandlerAndSelector(selectorOrXPath, (element, selector) => document.querySelector(selector));
+    const waitTask = new WaitTask(this, predicate, queryHandler, title, polling, timeout, updatedSelector, isXPath, waitForVisible, waitForHidden);
     const handle = await waitTask.promise;
     if (!handle.asElement()) {
       await handle.dispose();
@@ -397,7 +403,7 @@ export class DOMWorld {
     function predicate(selectorOrXPath: string, isXPath: boolean, waitForVisible: boolean, waitForHidden: boolean): Node | null | boolean {
       const node = isXPath
         ? document.evaluate(selectorOrXPath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue
-        : document.querySelector(selectorOrXPath);
+        : predicateQueryHandler ? predicateQueryHandler(document, selectorOrXPath) as Element : document.querySelector(selectorOrXPath);
       if (!node)
         return waitForHidden;
       if (!waitForVisible && !waitForHidden)
@@ -430,18 +436,31 @@ class WaitTask {
   _timeoutTimer?: NodeJS.Timeout;
   _terminated = false;
 
-  constructor(domWorld: DOMWorld, predicateBody: Function | string, title: string, polling: string | number, timeout: number, ...args: unknown[]) {
+  constructor(domWorld: DOMWorld, predicateBody: Function | string, predicateQueryHandlerBody: Function | string | undefined, title: string, polling: string | number, timeout: number, ...args: unknown[]) {
     if (helper.isString(polling))
       assert(polling === 'raf' || polling === 'mutation', 'Unknown polling option: ' + polling);
     else if (helper.isNumber(polling))
       assert(polling > 0, 'Cannot poll with non-positive interval: ' + polling);
     else
       throw new Error('Unknown polling options: ' + polling);
 
+    function getPredicateBody(predicateBody: Function | string, predicateQueryHandlerBody: Function | string) {
+      if (helper.isString(predicateBody))
+        return `return (${predicateBody});`;
+      if (predicateQueryHandlerBody) {
+        return `
+          return (function wrapper(args) {
+            const predicateQueryHandler = ${predicateQueryHandlerBody};
+            return (${predicateBody})(...args);
+          })(args);`;
+      }
+      return `return (${predicateBody})(...args);`;
+    }
+
     this._domWorld = domWorld;
     this._polling = polling;
     this._timeout = timeout;
-    this._predicateBody = helper.isString(predicateBody) ? 'return (' + predicateBody + ')' : 'return (' + predicateBody + ')(...args)';
+    this._predicateBody = getPredicateBody(predicateBody, predicateQueryHandlerBody);
     this._args = args;
     this._runCount = 0;
     domWorld._waitTasks.add(this);

src/FrameManager.ts

@@ -20,7 +20,7 @@ import {Events} from './Events';
 import {ExecutionContext, EVALUATION_SCRIPT_URL} from './ExecutionContext';
 import {LifecycleWatcher, PuppeteerLifeCycleEvent} from './LifecycleWatcher';
 import {DOMWorld, WaitForSelectorOptions} from './DOMWorld';
-import {NetworkManager} from './NetworkManager';
+import {NetworkManager, Response} from './NetworkManager';
 import {TimeoutSettings} from './TimeoutSettings';
 import {CDPSession} from './Connection';
 import {JSHandle, ElementHandle} from './JSHandle';
@@ -31,7 +31,7 @@ const UTILITY_WORLD_NAME = '__puppeteer_utility_world__';
 export class FrameManager extends EventEmitter {
   _client: CDPSession;
   _page: Puppeteer.Page;
-  _networkManager: Puppeteer.NetworkManager;
+  _networkManager: NetworkManager;
   _timeoutSettings: TimeoutSettings;
   _frames = new Map<string, Frame>();
   _contextIdToContext = new Map<number, ExecutionContext>();
@@ -70,11 +70,11 @@ export class FrameManager extends EventEmitter {
     ]);
   }
 
-  networkManager(): Puppeteer.NetworkManager {
+  networkManager(): NetworkManager {
     return this._networkManager;
   }
 
-  async navigateFrame(frame: Frame, url: string, options: {referer?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]} = {}): Promise<Puppeteer.Response | null> {
+  async navigateFrame(frame: Frame, url: string, options: {referer?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]} = {}): Promise<Response | null> {
     assertNoLegacyNavigationOptions(options);
     const {
       referer = this._networkManager.extraHTTPHeaders()['referer'],
@@ -110,7 +110,7 @@ export class FrameManager extends EventEmitter {
     }
   }
 
-  async waitForFrameNavigation(frame: Frame, options: {timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]} = {}): Promise<Puppeteer.Response | null> {
+  async waitForFrameNavigation(frame: Frame, options: {timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]} = {}): Promise<Response | null> {
     assertNoLegacyNavigationOptions(options);
     const {
       waitUntil = ['load'],
@@ -333,11 +333,11 @@ export class Frame {
       this._parentFrame._childFrames.add(this);
   }
 
-  async goto(url: string, options: {referer?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]}): Promise<Puppeteer.Response | null> {
+  async goto(url: string, options: {referer?: string; timeout?: number; waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[]}): Promise<Response | null> {
     return await this._frameManager.navigateFrame(this, url, options);
   }
 
-  async waitForNavigation(options: {timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]}): Promise<Puppeteer.Response | null> {
+  async waitForNavigation(options: {timeout?: number; waitUntil?: PuppeteerLifeCycleEvent|PuppeteerLifeCycleEvent[]}): Promise<Response | null> {
     return await this._frameManager.waitForFrameNavigation(this, options);
   }
 

src/JSHandle.ts

@@ -19,6 +19,7 @@ import {ExecutionContext} from './ExecutionContext';
 import {CDPSession} from './Connection';
 import {KeyInput} from './USKeyboardLayout';
 import {FrameManager, Frame} from './FrameManager';
+import {getQueryHandlerAndSelector} from './QueryHandler';
 
 interface BoxModel {
   content: Array<{x: number; y: number}>;
@@ -427,10 +428,10 @@ export class ElementHandle extends JSHandle {
   }
 
   async $(selector: string): Promise<ElementHandle | null> {
-    const handle = await this.evaluateHandle(
-        (element, selector) => element.querySelector(selector),
-        selector
-    );
+    const defaultHandler = (element: Element, selector: string) => element.querySelector(selector);
+    const {updatedSelector, queryHandler} = getQueryHandlerAndSelector(selector, defaultHandler);
+
+    const handle = await this.evaluateHandle(queryHandler, updatedSelector);
     const element = handle.asElement();
     if (element)
       return element;
@@ -443,10 +444,10 @@ export class ElementHandle extends JSHandle {
    * @return {!Promise<!Array<!ElementHandle>>}
    */
   async $$(selector: string): Promise<ElementHandle[]> {
-    const arrayHandle = await this.evaluateHandle(
-        (element, selector) => element.querySelectorAll(selector),
-        selector
-    );
+    const defaultHandler = (element: Element, selector: string) => element.querySelectorAll(selector);
+    const {updatedSelector, queryHandler} = getQueryHandlerAndSelector(selector, defaultHandler);
+
+    const arrayHandle = await this.evaluateHandle(queryHandler, updatedSelector);
     const properties = await arrayHandle.getProperties();
     await arrayHandle.dispose();
     const result = [];
@@ -468,11 +469,10 @@ export class ElementHandle extends JSHandle {
   }
 
   async $$eval<ReturnType extends any>(selector: string, pageFunction: Function | string, ...args: unknown[]): Promise<ReturnType> {
-    const arrayHandle = await this.evaluateHandle(
-        (element, selector) => Array.from(element.querySelectorAll(selector)),
-        selector
-    );
+    const defaultHandler = (element: Element, selector: string) => Array.from(element.querySelectorAll(selector));
+    const {updatedSelector, queryHandler} = getQueryHandlerAndSelector(selector, defaultHandler);
 
+    const arrayHandle = await this.evaluateHandle(queryHandler, updatedSelector);
     const result = await arrayHandle.evaluate<ReturnType>(pageFunction, ...args);
     await arrayHandle.dispose();
     return result;