Skip to content

Commit

Permalink
feat: use CDP's auto-attach mechanism (#8520)
Browse files Browse the repository at this point in the history 
* feat: use CDP's auto-attach mechanism

In this PR, we refactor Puppeteer to make use of the CDP's auto-attach mechanism. This allows the backend to pause
new targets and give Puppeteer a chance to configure them properly. This fixes the flakiness related to dealing with
OOPIFs and should fix some other issues related to the network interception and navigations. If those are not fixed completely by this PR, the PR serves a solid base for fixing them.

Closes #8507, #7990
Unlocks #3667

BREAKING CHANGE: With Chromium, Puppeteer will now attach to page/iframe targets immediately to allow reliable configuration of targets.
  • Loading branch information
@OrKoN @jrandolf
OrKoN authored and jrandolf committed Aug 2, 2022
1 parent a654607 commit 2cbfdeb
Show file tree
Hide file tree
Showing 20 changed files with 1,171 additions and 214 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Expand Up @@ -27,7 +27,7 @@
"node": ">=14.1.0"
},
"scripts": {
"test": "c8 --check-coverage --lines 94 run-s test:chrome test:chrome:* test:firefox",
"test": "c8 --check-coverage --lines 93 run-s test:chrome test:chrome:* test:firefox",
"test:types": "tsd",
"test:install": "scripts/test-install.sh",
"test:firefox": "cross-env PUPPETEER_PRODUCT=firefox MOZ_WEBRENDER=0 mocha",
Expand Down
179 changes: 118 additions & 61 deletions src/common/Browser.ts
View file
Expand Up @@ -17,13 +17,16 @@
import {ChildProcess} from 'child_process';
import {Protocol} from 'devtools-protocol';
import {assert} from './assert.js';
import {Connection, ConnectionEmittedEvents} from './Connection.js';
import {CDPSession, Connection, ConnectionEmittedEvents} from './Connection.js';
import {EventEmitter} from './EventEmitter.js';
import {waitWithTimeout} from './util.js';
import {Page} from './Page.js';
import {Viewport} from './PuppeteerViewport.js';
import {Target} from './Target.js';
import {TaskQueue} from './TaskQueue.js';
import {TargetManager, TargetManagerEmittedEvents} from './TargetManager.js';
import {ChromeTargetManager} from './ChromeTargetManager.js';
import {FirefoxTargetManager} from './FirefoxTargetManager.js';

/**
* BrowserContext options.
Expand Down Expand Up @@ -218,6 +221,7 @@ export class Browser extends EventEmitter {
* @internal
*/
static async _create(
product: 'firefox' | 'chrome' | undefined,
connection: Connection,
contextIds: string[],
ignoreHTTPSErrors: boolean,
Expand All @@ -228,6 +232,7 @@ export class Browser extends EventEmitter {
isPageTargetCallback?: IsPageTargetCallback
): Promise<Browser> {
const browser = new Browser(
product,
connection,
contextIds,
ignoreHTTPSErrors,
Expand All @@ -237,7 +242,7 @@ export class Browser extends EventEmitter {
targetFilterCallback,
isPageTargetCallback
);
await connection.send('Target.setDiscoverTargets', {discover: true});
await browser._attach();
return browser;
}
#ignoreHTTPSErrors: boolean;
Expand All @@ -250,20 +255,20 @@ export class Browser extends EventEmitter {
#defaultContext: BrowserContext;
#contexts: Map<string, BrowserContext>;
#screenshotTaskQueue: TaskQueue;
#targets: Map<string, Target>;
#ignoredTargets = new Set<string>();
#targetManager: TargetManager;

/**
* @internal
*/
get _targets(): Map<string, Target> {
return this.#targets;
return this.#targetManager.getAvailableTargets();
}

/**
* @internal
*/
constructor(
product: 'chrome' | 'firefox' | undefined,
connection: Connection,
contextIds: string[],
ignoreHTTPSErrors: boolean,
Expand All @@ -274,6 +279,7 @@ export class Browser extends EventEmitter {
isPageTargetCallback?: IsPageTargetCallback
) {
super();
product = product || 'chrome';
this.#ignoreHTTPSErrors = ignoreHTTPSErrors;
this.#defaultViewport = defaultViewport;
this.#process = process;
Expand All @@ -286,7 +292,19 @@ export class Browser extends EventEmitter {
return true;
});
this.#setIsPageTargetCallback(isPageTargetCallback);

if (product === 'firefox') {
this.#targetManager = new FirefoxTargetManager(
connection,
this.#createTarget,
this.#targetFilterCallback
);
} else {
this.#targetManager = new ChromeTargetManager(
connection,
this.#createTarget,
this.#targetFilterCallback
);
}
this.#defaultContext = new BrowserContext(this.#connection, this);
this.#contexts = new Map();
for (const contextId of contextIds) {
Expand All @@ -295,19 +313,62 @@ export class Browser extends EventEmitter {
new BrowserContext(this.#connection, this, contextId)
);
}
}

this.#targets = new Map();
this.#connection.on(ConnectionEmittedEvents.Disconnected, () => {
return this.emit(BrowserEmittedEvents.Disconnected);
});
this.#connection.on('Target.targetCreated', this.#targetCreated.bind(this));
#emitDisconnected = () => {
this.emit(BrowserEmittedEvents.Disconnected);
};

/**
* @internal
*/
async _attach(): Promise<void> {
this.#connection.on(
'Target.targetDestroyed',
this.#targetDestroyed.bind(this)
ConnectionEmittedEvents.Disconnected,
this.#emitDisconnected
);
this.#connection.on(
'Target.targetInfoChanged',
this.#targetInfoChanged.bind(this)
this.#targetManager.on(
TargetManagerEmittedEvents.TargetAvailable,
this.#onAttachedToTarget
);
this.#targetManager.on(
TargetManagerEmittedEvents.TargetGone,
this.#onDetachedFromTarget
);
this.#targetManager.on(
TargetManagerEmittedEvents.TargetChanged,
this.#onTargetChanged
);
this.#targetManager.on(
TargetManagerEmittedEvents.TargetDiscovered,
this.#onTargetDiscovered
);
await this.#targetManager.initialize();
}

/**
* @internal
*/
_detach(): void {
this.#connection.off(
ConnectionEmittedEvents.Disconnected,
this.#emitDisconnected
);
this.#targetManager.off(
TargetManagerEmittedEvents.TargetAvailable,
this.#onAttachedToTarget
);
this.#targetManager.off(
TargetManagerEmittedEvents.TargetGone,
this.#onDetachedFromTarget
);
this.#targetManager.off(
TargetManagerEmittedEvents.TargetChanged,
this.#onTargetChanged
);
this.#targetManager.off(
TargetManagerEmittedEvents.TargetDiscovered,
this.#onTargetDiscovered
);
}

Expand All @@ -319,6 +380,13 @@ export class Browser extends EventEmitter {
return this.#process ?? null;
}

/**
* @internal
*/
_targetManager(): TargetManager {
return this.#targetManager;
}

#setIsPageTargetCallback(isPageTargetCallback?: IsPageTargetCallback): void {
this.#isPageTargetCallback =
isPageTargetCallback ||
Expand Down Expand Up @@ -404,10 +472,10 @@ export class Browser extends EventEmitter {
this.#contexts.delete(contextId);
}

async #targetCreated(
event: Protocol.Target.TargetCreatedEvent
): Promise<void> {
const targetInfo = event.targetInfo;
#createTarget = (
targetInfo: Protocol.Target.TargetInfo,
session?: CDPSession
) => {
const {browserContextId} = targetInfo;
const context =
browserContextId && this.#contexts.has(browserContextId)
Expand All @@ -418,15 +486,11 @@ export class Browser extends EventEmitter {
throw new Error('Missing browser context');
}

const shouldAttachToTarget = this.#targetFilterCallback(targetInfo);
if (!shouldAttachToTarget) {
this.#ignoredTargets.add(targetInfo.targetId);
return;
}

const target = new Target(
return new Target(
targetInfo,
session,
context,
this.#targetManager,
() => {
return this.#connection.createSession(targetInfo);
},
Expand All @@ -435,59 +499,49 @@ export class Browser extends EventEmitter {
this.#screenshotTaskQueue,
this.#isPageTargetCallback
);
assert(
!this.#targets.has(event.targetInfo.targetId),
'Target should not exist before targetCreated'
);
this.#targets.set(event.targetInfo.targetId, target);
};

#onAttachedToTarget = async (target: Target) => {
if (await target._initializedPromise) {
this.emit(BrowserEmittedEvents.TargetCreated, target);
context.emit(BrowserContextEmittedEvents.TargetCreated, target);
target
.browserContext()
.emit(BrowserContextEmittedEvents.TargetCreated, target);
}
}
};

async #targetDestroyed(event: {targetId: string}): Promise<void> {
if (this.#ignoredTargets.has(event.targetId)) {
return;
}
const target = this.#targets.get(event.targetId);
if (!target) {
throw new Error(
`Missing target in _targetDestroyed (id = ${event.targetId})`
);
}
#onDetachedFromTarget = async (target: Target): Promise<void> => {
target._initializedCallback(false);
this.#targets.delete(event.targetId);
target._closedCallback();
if (await target._initializedPromise) {
this.emit(BrowserEmittedEvents.TargetDestroyed, target);
target
.browserContext()
.emit(BrowserContextEmittedEvents.TargetDestroyed, target);
}
}

#targetInfoChanged(event: Protocol.Target.TargetInfoChangedEvent): void {
if (this.#ignoredTargets.has(event.targetInfo.targetId)) {
return;
}
const target = this.#targets.get(event.targetInfo.targetId);
if (!target) {
throw new Error(
`Missing target in targetInfoChanged (id = ${event.targetInfo.targetId})`
);
}
};

#onTargetChanged = ({
target,
targetInfo,
}: {
target: Target;
targetInfo: Protocol.Target.TargetInfo;
}): void => {
const previousURL = target.url();
const wasInitialized = target._isInitialized;
target._targetInfoChanged(event.targetInfo);
target._targetInfoChanged(targetInfo);
if (wasInitialized && previousURL !== target.url()) {
this.emit(BrowserEmittedEvents.TargetChanged, target);
target
.browserContext()
.emit(BrowserContextEmittedEvents.TargetChanged, target);
}
}
};

#onTargetDiscovered = (targetInfo: Protocol.Target.TargetInfo): void => {
this.emit('targetdiscovered', targetInfo);
};

/**
* The browser websocket endpoint which can be used as an argument to
Expand Down Expand Up @@ -526,7 +580,7 @@ export class Browser extends EventEmitter {
url: 'about:blank',
browserContextId: contextId || undefined,
});
const target = this.#targets.get(targetId);
const target = this.#targetManager.getAvailableTargets().get(targetId);
if (!target) {
throw new Error(`Missing target for page (id = ${targetId})`);
}
Expand All @@ -548,7 +602,9 @@ export class Browser extends EventEmitter {
* an array with all the targets in all browser contexts.
*/
targets(): Target[] {
return Array.from(this.#targets.values()).filter(target => {
return Array.from(
this.#targetManager.getAvailableTargets().values()
).filter(target => {
return target._isInitialized;
});
}
Expand Down Expand Up @@ -671,6 +727,7 @@ export class Browser extends EventEmitter {
* cannot be used anymore.
*/
disconnect(): void {
this.#targetManager.dispose();
this.#connection.dispose();
}

Expand Down
9 changes: 7 additions & 2 deletions src/common/BrowserConnector.ts
View file
Expand Up @@ -26,7 +26,7 @@ import {Connection} from './Connection.js';
import {ConnectionTransport} from './ConnectionTransport.js';
import {getFetch} from './fetch.js';
import {Viewport} from './PuppeteerViewport.js';

import {Product} from './Product.js';
/**
* Generic browser options that can be passed when launching any browser or when
* connecting to an existing browser instance.
Expand Down Expand Up @@ -75,6 +75,7 @@ export async function _connectToBrowser(
browserWSEndpoint?: string;
browserURL?: string;
transport?: ConnectionTransport;
product?: Product;
}
): Promise<Browser> {
const {
Expand All @@ -86,6 +87,7 @@ export async function _connectToBrowser(
slowMo = 0,
targetFilter,
_isPageTarget: isPageTarget,
product,
} = options;

assert(
Expand Down Expand Up @@ -113,7 +115,8 @@ export async function _connectToBrowser(
const {browserContextIds} = await connection.send(
'Target.getBrowserContexts'
);
return Browser._create(
const browser = await Browser._create(
product || 'chrome',
connection,
browserContextIds,
ignoreHTTPSErrors,
Expand All @@ -125,6 +128,8 @@ export async function _connectToBrowser(
targetFilter,
isPageTarget
);
await browser.pages();
return browser;
}

async function getWSEndpoint(browserURL: string): Promise<string> {
Expand Down

0 comments on commit 2cbfdeb

Please sign in to comment.