From ce894a2ffce4bc44bd11f12d1f0543e003a97e02 Mon Sep 17 00:00:00 2001 From: Alex Rudenko Date: Thu, 25 Jan 2024 21:39:07 +0100 Subject: [PATCH] feat: download chrome-headless-shell by default and use it for the old headless mode (#11754) --- README.md | 4 +- docs/api/puppeteer.configuration.md | 26 ++--- docs/index.md | 4 +- .../src/common/Configuration.ts | 12 +++ .../puppeteer-core/src/node/ChromeLauncher.ts | 9 +- .../src/node/ProductLauncher.ts | 9 +- packages/puppeteer-core/src/revisions.ts | 1 + packages/puppeteer/src/getConfiguration.ts | 18 ++++ packages/puppeteer/src/node/cli.ts | 1 + packages/puppeteer/src/node/install.ts | 95 +++++++++++++++---- .../src/puppeteer-configuration.spec.ts | 10 +- test/installation/src/puppeteer.spec.ts | 11 ++- tools/update_chrome_revision.mjs | 2 +- 13 files changed, 155 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 117cbdd64bdc1..74a15c6eb9d11 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,10 @@ pnpm i puppeteer ``` When you install Puppeteer, it automatically downloads a recent version of -[Chrome for Testing](https://developer.chrome.com/blog/chrome-for-testing/) (~170MB macOS, ~282MB Linux, ~280MB Windows) that is [guaranteed to +[Chrome for Testing](https://developer.chrome.com/blog/chrome-for-testing/) (~170MB macOS, ~282MB Linux, ~280MB Windows) and a `chrome-headless-shell` binary (starting with Puppeteer v21.6.0) that is [guaranteed to work](https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy) with Puppeteer. The browser is downloaded to the `$HOME/.cache/puppeteer` folder -by default (starting with Puppeteer v19.0.0). +by default (starting with Puppeteer v19.0.0). See [configuration](https://pptr.dev/api/puppeteer.configuration) for configuration options and environmental variables to control the download behavor. If you deploy a project using Puppeteer to a hosting provider, such as Render or Heroku, you might need to reconfigure the location of the cache to be within diff --git a/docs/api/puppeteer.configuration.md b/docs/api/puppeteer.configuration.md index c8a49caacc700..8f80db8851690 100644 --- a/docs/api/puppeteer.configuration.md +++ b/docs/api/puppeteer.configuration.md @@ -16,15 +16,17 @@ export interface Configuration ## Properties -| Property | Modifiers | Type | Description | Default | -| ------------------ | --------------------- | ------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| browserRevision | optional | string |

Specifies a certain version of the browser you'd like Puppeteer to use.

Can be overridden by PUPPETEER_BROWSER_REVISION.

See [puppeteer.launch](./puppeteer.puppeteernode.launch.md) on how executable path is inferred.

| A compatible-revision of the browser. | -| cacheDirectory | optional | string |

Defines the directory to be used by Puppeteer for caching.

Can be overridden by PUPPETEER_CACHE_DIR.

| path.join(os.homedir(), '.cache', 'puppeteer') | -| defaultProduct | optional | [Product](./puppeteer.product.md) |

Specifies which browser you'd like Puppeteer to use.

Can be overridden by PUPPETEER_PRODUCT.

| chrome | -| downloadBaseUrl | optional | string |

Specifies the URL prefix that is used to download the browser.

Can be overridden by PUPPETEER_DOWNLOAD_BASE_URL.

| Either https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing or https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central, depending on the product. | -| downloadPath | optional | string |

Specifies the path for the downloads folder.

Can be overridden by PUPPETEER_DOWNLOAD_PATH.

| <cacheDirectory> | -| executablePath | optional | string |

Specifies an executable path to be used in [puppeteer.launch](./puppeteer.puppeteernode.launch.md).

Can be overridden by PUPPETEER_EXECUTABLE_PATH.

| **Auto-computed.** | -| experiments | optional | [ExperimentsConfiguration](./puppeteer.experimentsconfiguration.md) | Defines experimental options for Puppeteer. | | -| logLevel | optional | 'silent' \| 'error' \| 'warn' | Tells Puppeteer to log at the given level. | warn | -| skipDownload | optional | boolean |

Tells Puppeteer to not download during installation.

Can be overridden by PUPPETEER_SKIP_DOWNLOAD.

| | -| temporaryDirectory | optional | string |

Defines the directory to be used by Puppeteer for creating temporary files.

Can be overridden by PUPPETEER_TMP_DIR.

| os.tmpdir() | +| Property | Modifiers | Type | Description | Default | +| ------------------------------- | --------------------- | ------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| browserRevision | optional | string |

Specifies a certain version of the browser you'd like Puppeteer to use.

Can be overridden by PUPPETEER_BROWSER_REVISION.

See [puppeteer.launch](./puppeteer.puppeteernode.launch.md) on how executable path is inferred.

| A compatible-revision of the browser. | +| cacheDirectory | optional | string |

Defines the directory to be used by Puppeteer for caching.

Can be overridden by PUPPETEER_CACHE_DIR.

| path.join(os.homedir(), '.cache', 'puppeteer') | +| defaultProduct | optional | [Product](./puppeteer.product.md) |

Specifies which browser you'd like Puppeteer to use.

Can be overridden by PUPPETEER_PRODUCT.

| chrome | +| downloadBaseUrl | optional | string |

Specifies the URL prefix that is used to download the browser.

Can be overridden by PUPPETEER_DOWNLOAD_BASE_URL.

| Either https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing or https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central, depending on the product. | +| downloadPath | optional | string |

Specifies the path for the downloads folder.

Can be overridden by PUPPETEER_DOWNLOAD_PATH.

| <cacheDirectory> | +| executablePath | optional | string |

Specifies an executable path to be used in [puppeteer.launch](./puppeteer.puppeteernode.launch.md).

Can be overridden by PUPPETEER_EXECUTABLE_PATH.

| **Auto-computed.** | +| experiments | optional | [ExperimentsConfiguration](./puppeteer.experimentsconfiguration.md) | Defines experimental options for Puppeteer. | | +| logLevel | optional | 'silent' \| 'error' \| 'warn' | Tells Puppeteer to log at the given level. | warn | +| skipChromeDownload | optional | boolean |

Tells Puppeteer to not Chrome download during installation.

Can be overridden by PUPPETEER_SKIP_CHROME_DOWNLOAD.

| | +| skipChromeHeadlessShellDownload | optional | boolean |

Tells Puppeteer to not chrome-headless-shell download during installation.

Can be overridden by PUPPETEER_SKIP_CHROME_HEADLESSS_HELL_DOWNLOAD.

| | +| skipDownload | optional | boolean |

Tells Puppeteer to not download during installation.

Can be overridden by PUPPETEER_SKIP_DOWNLOAD.

| | +| temporaryDirectory | optional | string |

Defines the directory to be used by Puppeteer for creating temporary files.

Can be overridden by PUPPETEER_TMP_DIR.

| os.tmpdir() | diff --git a/docs/index.md b/docs/index.md index 117cbdd64bdc1..74a15c6eb9d11 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,10 +46,10 @@ pnpm i puppeteer ``` When you install Puppeteer, it automatically downloads a recent version of -[Chrome for Testing](https://developer.chrome.com/blog/chrome-for-testing/) (~170MB macOS, ~282MB Linux, ~280MB Windows) that is [guaranteed to +[Chrome for Testing](https://developer.chrome.com/blog/chrome-for-testing/) (~170MB macOS, ~282MB Linux, ~280MB Windows) and a `chrome-headless-shell` binary (starting with Puppeteer v21.6.0) that is [guaranteed to work](https://pptr.dev/faq#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy) with Puppeteer. The browser is downloaded to the `$HOME/.cache/puppeteer` folder -by default (starting with Puppeteer v19.0.0). +by default (starting with Puppeteer v19.0.0). See [configuration](https://pptr.dev/api/puppeteer.configuration) for configuration options and environmental variables to control the download behavor. If you deploy a project using Puppeteer to a hosting provider, such as Render or Heroku, you might need to reconfigure the location of the cache to be within diff --git a/packages/puppeteer-core/src/common/Configuration.ts b/packages/puppeteer-core/src/common/Configuration.ts index 9a677aa02f764..c64d109a7c5b9 100644 --- a/packages/puppeteer-core/src/common/Configuration.ts +++ b/packages/puppeteer-core/src/common/Configuration.ts @@ -95,6 +95,18 @@ export interface Configuration { * Can be overridden by `PUPPETEER_SKIP_DOWNLOAD`. */ skipDownload?: boolean; + /** + * Tells Puppeteer to not Chrome download during installation. + * + * Can be overridden by `PUPPETEER_SKIP_CHROME_DOWNLOAD`. + */ + skipChromeDownload?: boolean; + /** + * Tells Puppeteer to not chrome-headless-shell download during installation. + * + * Can be overridden by `PUPPETEER_SKIP_CHROME_HEADLESSS_HELL_DOWNLOAD`. + */ + skipChromeHeadlessShellDownload?: boolean; /** * Tells Puppeteer to log at the given level. * diff --git a/packages/puppeteer-core/src/node/ChromeLauncher.ts b/packages/puppeteer-core/src/node/ChromeLauncher.ts index 474469c076109..51d5a1998324a 100644 --- a/packages/puppeteer-core/src/node/ChromeLauncher.ts +++ b/packages/puppeteer-core/src/node/ChromeLauncher.ts @@ -146,7 +146,7 @@ export class ChromeLauncher extends ProductLauncher { channel || !this.puppeteer._isPuppeteerCore, `An \`executablePath\` or \`channel\` must be specified for \`puppeteer-core\`` ); - chromeExecutable = this.executablePath(channel); + chromeExecutable = this.executablePath(channel, options.headless ?? true); } return { @@ -269,14 +269,17 @@ export class ChromeLauncher extends ProductLauncher { return chromeArguments; } - override executablePath(channel?: ChromeReleaseChannel): string { + override executablePath( + channel?: ChromeReleaseChannel, + headless?: boolean | 'new' + ): string { if (channel) { return computeSystemExecutablePath({ browser: SupportedBrowsers.CHROME, channel: convertPuppeteerChannelToBrowsersChannel(channel), }); } else { - return this.resolveExecutablePath(); + return this.resolveExecutablePath(headless); } } } diff --git a/packages/puppeteer-core/src/node/ProductLauncher.ts b/packages/puppeteer-core/src/node/ProductLauncher.ts index 8d532f428363e..ab3432cd3afc9 100644 --- a/packages/puppeteer-core/src/node/ProductLauncher.ts +++ b/packages/puppeteer-core/src/node/ProductLauncher.ts @@ -393,7 +393,7 @@ export abstract class ProductLauncher { /** * @internal */ - protected resolveExecutablePath(): string { + protected resolveExecutablePath(headless?: boolean | 'new'): string { let executablePath = this.puppeteer.configuration.executablePath; if (executablePath) { if (!existsSync(executablePath)) { @@ -404,9 +404,12 @@ export abstract class ProductLauncher { return executablePath; } - function productToBrowser(product?: Product) { + function productToBrowser(product?: Product, headless?: boolean | 'new') { switch (product) { case 'chrome': + if (headless === true) { + return InstalledBrowser.CHROMEHEADLESSSHELL; + } return InstalledBrowser.CHROME; case 'firefox': return InstalledBrowser.FIREFOX; @@ -416,7 +419,7 @@ export abstract class ProductLauncher { executablePath = computeExecutablePath({ cacheDir: this.puppeteer.defaultDownloadPath!, - browser: productToBrowser(this.product), + browser: productToBrowser(this.product, headless), buildId: this.puppeteer.browserRevision, }); diff --git a/packages/puppeteer-core/src/revisions.ts b/packages/puppeteer-core/src/revisions.ts index 49d1e4e634423..37360204d8aec 100644 --- a/packages/puppeteer-core/src/revisions.ts +++ b/packages/puppeteer-core/src/revisions.ts @@ -9,5 +9,6 @@ */ export const PUPPETEER_REVISIONS = Object.freeze({ chrome: '121.0.6167.85', + 'chrome-headless-shell': '121.0.6167.85', firefox: 'latest', }); diff --git a/packages/puppeteer/src/getConfiguration.ts b/packages/puppeteer/src/getConfiguration.ts index 5f2085d8b38a3..28cf026eb72df 100644 --- a/packages/puppeteer/src/getConfiguration.ts +++ b/packages/puppeteer/src/getConfiguration.ts @@ -64,6 +64,24 @@ export const getConfiguration = (): Configuration => { configuration.skipDownload ); + // Set skipChromeDownload explicitly or from default + configuration.skipChromeDownload = Boolean( + process.env['PUPPETEER_SKIP_CHROME_DOWNLOAD'] ?? + process.env['npm_config_puppeteer_skip_chrome_download'] ?? + process.env['npm_package_config_puppeteer_skip_chrome_download'] ?? + configuration.skipChromeDownload + ); + + // Set skipChromeDownload explicitly or from default + configuration.skipChromeHeadlessShellDownload = Boolean( + process.env['PUPPETEER_SKIP_CHROME_HEADLESS_SHELL_DOWNLOAD'] ?? + process.env['npm_config_puppeteer_skip_chrome_headless_shell_download'] ?? + process.env[ + 'npm_package_config_puppeteer_skip_chrome_headless_shell_download' + ] ?? + configuration.skipChromeHeadlessShellDownload + ); + // Prepare variables used in browser downloading if (!configuration.skipDownload) { configuration.browserRevision = diff --git a/packages/puppeteer/src/node/cli.ts b/packages/puppeteer/src/node/cli.ts index c846cfeed8d96..9a25c59327459 100644 --- a/packages/puppeteer/src/node/cli.ts +++ b/packages/puppeteer/src/node/cli.ts @@ -27,5 +27,6 @@ void new CLI({ pinnedBrowsers: { [Browser.CHROME]: PUPPETEER_REVISIONS.chrome, [Browser.FIREFOX]: PUPPETEER_REVISIONS.firefox, + [Browser.CHROMEHEADLESSSHELL]: PUPPETEER_REVISIONS['chrome-headless-shell'], }, }).run(process.argv); diff --git a/packages/puppeteer/src/node/install.ts b/packages/puppeteer/src/node/install.ts index 526148e2c8678..76bad868b8ed8 100644 --- a/packages/puppeteer/src/node/install.ts +++ b/packages/puppeteer/src/node/install.ts @@ -48,28 +48,91 @@ export async function downloadBrowser(): Promise { const unresolvedBuildId = configuration.browserRevision || PUPPETEER_REVISIONS[product] || 'latest'; + const unresolvedShellBuildId = + configuration.browserRevision || + PUPPETEER_REVISIONS['chrome-headless-shell'] || + 'latest'; - const buildId = await resolveBuildId(browser, platform, unresolvedBuildId); // TODO: deprecate downloadPath in favour of cacheDirectory. const cacheDir = configuration.downloadPath ?? configuration.cacheDirectory!; try { - const result = await install({ - browser, - cacheDir, - platform, - buildId, - downloadProgressCallback: makeProgressCallback(browser, buildId), - baseUrl: downloadBaseUrl, - }); - - logPolitely( - `${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}` - ); + const installationJobs = []; + + if (configuration.skipChromeDownload) { + logPolitely('**INFO** Skipping Chrome download as instructed.'); + } else { + const buildId = await resolveBuildId( + browser, + platform, + unresolvedBuildId + ); + installationJobs.push( + install({ + browser, + cacheDir, + platform, + buildId, + downloadProgressCallback: makeProgressCallback(browser, buildId), + baseUrl: downloadBaseUrl, + }) + .then(result => { + logPolitely( + `${supportedProducts[product]} (${result.buildId}) downloaded to ${result.path}` + ); + }) + .catch(error => { + throw new Error( + `ERROR: Failed to set up ${supportedProducts[product]} v${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`, + { + cause: error, + } + ); + }) + ); + } + + if (browser === Browser.CHROME) { + if (configuration.skipChromeHeadlessShellDownload) { + logPolitely('**INFO** Skipping Chrome download as instructed.'); + } else { + const shellBuildId = await resolveBuildId( + browser, + platform, + unresolvedShellBuildId + ); + + installationJobs.push( + install({ + browser: Browser.CHROMEHEADLESSSHELL, + cacheDir, + platform, + buildId: shellBuildId, + downloadProgressCallback: makeProgressCallback( + browser, + shellBuildId + ), + baseUrl: downloadBaseUrl, + }) + .then(result => { + logPolitely( + `${Browser.CHROMEHEADLESSSHELL} (${result.buildId}) downloaded to ${result.path}` + ); + }) + .catch(error => { + throw new Error( + `ERROR: Failed to set up ${Browser.CHROMEHEADLESSSHELL} v${shellBuildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`, + { + cause: error, + } + ); + }) + ); + } + } + + await Promise.all(installationJobs); } catch (error) { - console.error( - `ERROR: Failed to set up ${supportedProducts[product]} r${buildId}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` - ); console.error(error); process.exit(1); } diff --git a/test/installation/src/puppeteer-configuration.spec.ts b/test/installation/src/puppeteer-configuration.spec.ts index 70d77bedd4847..1ed5511f6c1c2 100644 --- a/test/installation/src/puppeteer-configuration.spec.ts +++ b/test/installation/src/puppeteer-configuration.spec.ts @@ -30,8 +30,9 @@ describe('`puppeteer` with configuration', () => { it('evaluates', async function () { const files = await readdir(join(this.sandbox, '.cache', 'puppeteer')); - assert.equal(files.length, 1); - assert.equal(files[0], 'chrome'); + assert.equal(files.length, 2); + assert(files.includes('chrome')); + assert(files.includes('chrome-headless-shell')); const script = await readAsset('puppeteer', 'basic.js'); await this.runScript(script, 'mjs'); @@ -61,8 +62,9 @@ describe('`puppeteer` with configuration', () => { it('evaluates', async function () { const files = await readdir(join(this.sandbox, '.cache', 'puppeteer')); - assert.equal(files.length, 1); - assert.equal(files[0], 'chrome'); + assert.equal(files.length, 2); + assert(files.includes('chrome')); + assert(files.includes('chrome-headless-shell')); const script = await readAsset('puppeteer', 'basic.js'); await this.runScript(script, 'mjs'); diff --git a/test/installation/src/puppeteer.spec.ts b/test/installation/src/puppeteer.spec.ts index db78170f55d95..d7b8757284db7 100644 --- a/test/installation/src/puppeteer.spec.ts +++ b/test/installation/src/puppeteer.spec.ts @@ -25,8 +25,10 @@ describe('`puppeteer`', () => { it('evaluates CommonJS', async function () { const files = await readdir(join(this.sandbox, '.cache', 'puppeteer')); - assert.equal(files.length, 1); - assert.equal(files[0], 'chrome'); + assert.equal(files.length, 2); + assert(files.includes('chrome')); + assert(files.includes('chrome-headless-shell')); + const script = await readAsset('puppeteer-core', 'requires.cjs'); await this.runScript(script, 'cjs'); }); @@ -52,8 +54,9 @@ describe('`puppeteer`', () => { it('evaluates', async function () { const files = await readdir(join(this.sandbox, '.cache', 'puppeteer')); - assert.equal(files.length, 1); - assert.equal(files[0], 'chrome'); + assert.equal(files.length, 2); + assert(files.includes('chrome')); + assert(files.includes('chrome-headless-shell')); const script = await readAsset('puppeteer', 'basic.js'); await this.runScript(script, 'mjs'); diff --git a/tools/update_chrome_revision.mjs b/tools/update_chrome_revision.mjs index 6e8580a1de257..64eeef74d5808 100644 --- a/tools/update_chrome_revision.mjs +++ b/tools/update_chrome_revision.mjs @@ -62,7 +62,7 @@ async function formatUpdateFiles() { async function replaceInFile(filePath, search, replace) { const buffer = await readFile(filePath); - const update = buffer.toString().replace(search, replace); + const update = buffer.toString().replaceAll(search, replace); await writeFile(filePath, update);