From 33f1967072e07824c5bf6a8c1336f844d9efaabf Mon Sep 17 00:00:00 2001 From: Maja Frydrychowicz Date: Tue, 10 Mar 2020 16:59:03 -0400 Subject: [PATCH] (feat) Add option to fetch Firefox Nightly (#5467) * (feat) Add option to fetch Firefox Nightly Add Firefox support to BrowserFetcher and the install script. By default, the latest Firefox Nightly is downloaded directly from archive.mozilla.org (dmg, tar.bz2 and zip) This also required changes that impact `puppeteer.launch()` and `puppeteer.executablePath()` Fixes #5151 * Update docs/api.md Co-Authored-By: Mathias Bynens * Clean up revision promise * Improve error handling in revision check * Remove matchAll * Use explicit octal mode * Update .gitignore Co-authored-by: Mathias Bynens --- .gitignore | 1 + .npmignore | 1 + README.md | 22 +- docs/api.md | 31 ++- install.js | 252 +++++++++++------- lib/BrowserFetcher.js | 235 +++++++++++++--- lib/Launcher.js | 26 +- lib/Puppeteer.js | 19 +- package.json | 6 +- .../firefox-75.0a1.en-US.linux-x86_64.tar.bz2 | Bin 0 -> 211 bytes test/launcher.spec.js | 51 +++- 11 files changed, 470 insertions(+), 174 deletions(-) create mode 100644 test/assets/firefox-75.0a1.en-US.linux-x86_64.tar.bz2 diff --git a/.gitignore b/.gitignore index aae2a3e93c8da..ebdd78888fcd5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /test/output-firefox /test/test-user-data-dir* /.local-chromium/ +/.local-firefox/ /.dev_profile* .DS_Store *.swp diff --git a/.npmignore b/.npmignore index 509cac286849b..b24ba6246e4c6 100644 --- a/.npmignore +++ b/.npmignore @@ -14,6 +14,7 @@ utils/node6-transform # repeats from .gitignore node_modules .local-chromium +.local-firefox .dev_profile* .DS_Store *.swp diff --git a/README.md b/README.md index f4f3196c7ab84..3a2b70025ff20 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,13 @@ npm i puppeteer # or "yarn add puppeteer" ``` -Note: When you install Puppeteer, it downloads a recent version of Chromium (~170MB Mac, ~282MB Linux, ~280MB Win) that is guaranteed to work with the API. To skip the download, see [Environment variables](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#environment-variables). +Note: When you install Puppeteer, it downloads a recent version of Chromium (~170MB Mac, ~282MB Linux, ~280MB Win) that is guaranteed to work with the API. To skip the download, or to download a different browser, see [Environment variables](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#environment-variables). ### puppeteer-core Since version 1.7.0 we publish the [`puppeteer-core`](https://www.npmjs.com/package/puppeteer-core) package, -a version of Puppeteer that doesn't download Chromium by default. +a version of Puppeteer that doesn't download any browser by default. ```bash npm i puppeteer-core @@ -173,13 +173,13 @@ pass in the executable's path when creating a `Browser` instance: const browser = await puppeteer.launch({executablePath: '/path/to/Chrome'}); ``` -See [`Puppeteer.launch()`](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#puppeteerlaunchoptions) for more information. +You can also use Puppeteer with Firefox Nightly (experimental support). See [`Puppeteer.launch()`](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#puppeteerlaunchoptions) for more information. See [`this article`](https://www.howtogeek.com/202825/what%E2%80%99s-the-difference-between-chromium-and-chrome/) for a description of the differences between Chromium and Chrome. [`This article`](https://chromium.googlesource.com/chromium/src/+/master/docs/chromium_browser_vs_google_chrome.md) describes some differences for Linux users. **3. Creates a fresh user profile** -Puppeteer creates its own Chromium user profile which it **cleans up on every run**. +Puppeteer creates its own browser user profile which it **cleans up on every run**. @@ -301,7 +301,7 @@ See [Contributing](https://github.com/puppeteer/puppeteer/blob/master/CONTRIBUTI Historically, Puppeteer supported Firefox indirectly through puppeteer-firefox, which relied on a custom, patched version of Firefox. This approach was also known as “Juggler”. After discussions with Mozilla, we collectively concluded that relying on custom patches was infeasible. Since then, we have been collaborating with Mozilla on supporting Puppeteer on “stock” Firefox. -From Puppeteer v2.1.0 onwards, as an experimental feature, you can specify [`puppeteer.launch({product: 'firefox'})`](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#puppeteerlaunchoptions) to run your Puppeteer scripts in Firefox, without any additional custom patches. +From Puppeteer v2.1.0 onwards, as an experimental feature, you can specify [`puppeteer.launch({product: 'firefox'})`](https://github.com/puppeteer/puppeteer/blob/v2.1.1/docs/api.md#puppeteerlaunchoptions) to run your Puppeteer scripts in Firefox Nightly, without any additional custom patches. We will continue collaborating with other browser vendors to bring Puppeteer support to browsers such as Safari. This effort includes exploration of a standard for executing cross-browser commands (instead of relying on the non-standard DevTools Protocol used by Chrome). @@ -356,6 +356,18 @@ npm install puppeteer-core@chrome-71 Look for `chromium_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/master/package.json). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section. + +#### Q: Which Firefox version does Puppeteer use? + +Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/master/package.json) is `latest` -- Puppeteer isn't tied to a particular Firefox version. + +To fetch Firefox Nightly as part of Puppeteer installation: + +```bash +PUPPETEER_PRODUCT=firefox npm i puppeteer +# or "yarn add puppeteer" +``` + #### Q: What’s considered a “Navigation”? From Puppeteer’s standpoint, **“navigation” is anything that changes a page’s URL**. diff --git a/docs/api.md b/docs/api.md index f8ef097da1a07..87fa62130e40b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -36,8 +36,10 @@ - [class: BrowserFetcher](#class-browserfetcher) * [browserFetcher.canDownload(revision)](#browserfetchercandownloadrevision) * [browserFetcher.download(revision[, progressCallback])](#browserfetcherdownloadrevision-progresscallback) + * [browserFetcher.host()](#browserfetcherhost) * [browserFetcher.localRevisions()](#browserfetcherlocalrevisions) * [browserFetcher.platform()](#browserfetcherplatform) + * [browserFetcher.product()](#browserfetcherproduct) * [browserFetcher.remove(revision)](#browserfetcherremoverevision) * [browserFetcher.revisionInfo(revision)](#browserfetcherrevisioninforevision) - [class: Browser](#class-browser) @@ -391,7 +393,7 @@ If Puppeteer doesn't find them in the environment during the installation step, - `PUPPETEER_DOWNLOAD_HOST` - overwrite URL prefix that is used to download Chromium. Note: this includes protocol and might even include path prefix. Defaults to `https://storage.googleapis.com`. - `PUPPETEER_CHROMIUM_REVISION` - specify a certain version of Chromium you'd like Puppeteer to use. See [puppeteer.launch([options])](#puppeteerlaunchoptions) on how executable path is inferred. **BEWARE**: Puppeteer is only [guaranteed to work](https://github.com/puppeteer/puppeteer/#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy) with the bundled Chromium, use at your own risk. - `PUPPETEER_EXECUTABLE_PATH` - specify an executable path to be used in `puppeteer.launch`. See [puppeteer.launch([options])](#puppeteerlaunchoptions) on how the executable path is inferred. **BEWARE**: Puppeteer is only [guaranteed to work](https://github.com/puppeteer/puppeteer/#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy) with the bundled Chromium, use at your own risk. -- `PUPPETEER_PRODUCT` - specify which browser you'd like Puppeteer to use. Must be one of `chrome` or `firefox`. Setting `product` programmatically in [puppeteer.launch([options])](#puppeteerlaunchoptions) supercedes this environment variable. The product is exposed in [`puppeteer.product`](#puppeteerproduct) +- `PUPPETEER_PRODUCT` - specify which browser you'd like Puppeteer to use. Must be one of `chrome` or `firefox`. This can also be used during installation to fetch the recommended browser binary. Setting `product` programmatically in [puppeteer.launch([options])](#puppeteerlaunchoptions) supersedes this environment variable. The product is exposed in [`puppeteer.product`](#puppeteerproduct) > **NOTE** PUPPETEER_* env variables are not accounted for in the [`puppeteer-core`](https://www.npmjs.com/package/puppeteer-core) package. @@ -461,9 +463,10 @@ This methods attaches Puppeteer to an existing Chromium instance. #### puppeteer.createBrowserFetcher([options]) - `options` <[Object]> - - `host` <[string]> A download host to be used. Defaults to `https://storage.googleapis.com`. - - `path` <[string]> A path for the downloads folder. Defaults to `/.local-chromium`, where `` is puppeteer's package root. + - `host` <[string]> A download host to be used. Defaults to `https://storage.googleapis.com`. If the `product` is `firefox`, this defaults to `https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central`. + - `path` <[string]> A path for the downloads folder. Defaults to `/.local-chromium`, where `` is puppeteer's package root. If the `product` is `firefox`, this defaults to `/.local-firefox`. - `platform` <[string]> Possible values are: `mac`, `win32`, `win64`, `linux`. Defaults to the current platform. + - `product` <[string]> Possible values are: `chrome`, `firefox`. Defaults to `chrome`. - returns: <[BrowserFetcher]> #### puppeteer.defaultArgs([options]) @@ -522,7 +525,7 @@ try { > **NOTE** The old way (Puppeteer versions <= v1.14.0) errors can be obtained with `require('puppeteer/Errors')`. #### puppeteer.executablePath() -- returns: <[string]> A path where Puppeteer expects to find bundled Chromium. Chromium might not exist there if the download was skipped with [`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`](#environment-variables). +- returns: <[string]> A path where Puppeteer expects to find the bundled browser. The browser binary might not be there if the download was skipped with [`PUPPETEER_SKIP_DOWNLOAD`](#environment-variables). > **NOTE** `puppeteer.executablePath()` is affected by the `PUPPETEER_EXECUTABLE_PATH` and `PUPPETEER_CHROMIUM_REVISION` env variables. See [Environment Variables](#environment-variables) for details. @@ -572,17 +575,19 @@ const browser = await puppeteer.launch({ > See [`this article`](https://www.howtogeek.com/202825/what%E2%80%99s-the-difference-between-chromium-and-chrome/) for a description of the differences between Chromium and Chrome. [`This article`](https://chromium.googlesource.com/chromium/src/+/lkgr/docs/chromium_browser_vs_google_chrome.md) describes some differences for Linux users. #### puppeteer.product -- returns: <[string]> returns the name of the browser that is under automation ("chrome" or "firefox") +- returns: <[string]> returns the name of the browser that is under automation (`"chrome"` or `"firefox"`) The product is set by the `PUPPETEER_PRODUCT` environment variable or the `product` option in [puppeteer.launch([options])](#puppeteerlaunchoptions) and defaults to `chrome`. Firefox support is experimental. ### class: BrowserFetcher -BrowserFetcher can download and manage different versions of Chromium. +BrowserFetcher can download and manage different versions of Chromium and Firefox. BrowserFetcher operates on revision strings that specify a precise version of Chromium, e.g. `"533271"`. Revision strings can be obtained from [omahaproxy.appspot.com](http://omahaproxy.appspot.com/). +In the Firefox case, BrowserFetcher downloads Firefox Nightly and operates on version numbers such as `"75"`. + An example of using BrowserFetcher to download a specific version of Chromium and running Puppeteer against it: @@ -615,14 +620,20 @@ The method initiates a HEAD request to check if the revision is available. The method initiates a GET request to download the revision from the host. +#### browserFetcher.host() +- returns: <[string]> The download host being used. + #### browserFetcher.localRevisions() -- returns: <[Promise]<[Array]<[string]>>> A list of all revisions available locally on disk. +- returns: <[Promise]<[Array]<[string]>>> A list of all revisions (for the current `product`) available locally on disk. #### browserFetcher.platform() - returns: <[string]> One of `mac`, `linux`, `win32` or `win64`. +#### browserFetcher.product() +- returns: <[string]> One of `chrome` or `firefox`. + #### browserFetcher.remove(revision) -- `revision` <[string]> a revision to remove. The method will throw if the revision has not been downloaded. +- `revision` <[string]> a revision to remove for the current `product`. The method will throw if the revision has not been downloaded. - returns: <[Promise]> Resolves when the revision has been removed. #### browserFetcher.revisionInfo(revision) @@ -633,6 +644,10 @@ The method initiates a GET request to download the revision from the host. - `executablePath` <[string]> path to the revision executable - `url` <[string]> URL this revision can be downloaded from - `local` <[boolean]> whether the revision is locally available on disk + - `product` <[string]> one of `chrome` or `firefox` + +> **NOTE** Many BrowserFetcher methods, like `remove` and `revisionInfo` +> are affected by the choice of `product`. See [puppeteer.createBrowserFetcher([options])](#puppeteercreatebrowserfetcheroptions). ### class: Browser diff --git a/install.js b/install.js index 16974266470a6..21bf712e6ec44 100644 --- a/install.js +++ b/install.js @@ -20,105 +20,146 @@ * By default, the `puppeteer` package runs this script during the installation * process unless one of the env flags is provided. * `puppeteer-core` package doesn't include this step at all. However, it's - * still possible to install Chromium using this script when necessary. + * still possible to install a supported browser using this script when + * necessary. */ -if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { - logPolitely('**INFO** Skipping Chromium download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.'); - return; -} -if (process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || process.env.npm_config_puppeteer_skip_chromium_download) { - logPolitely('**INFO** Skipping Chromium download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.'); - return; -} -if (process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || process.env.npm_package_config_puppeteer_skip_chromium_download) { - logPolitely('**INFO** Skipping Chromium download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.'); - return; -} - -const downloadHost = process.env.PUPPETEER_DOWNLOAD_HOST || process.env.npm_config_puppeteer_download_host || process.env.npm_package_config_puppeteer_download_host; - -const puppeteer = require('./index'); -const browserFetcher = puppeteer.createBrowserFetcher({ host: downloadHost }); - -const revision = process.env.PUPPETEER_CHROMIUM_REVISION || process.env.npm_config_puppeteer_chromium_revision || process.env.npm_package_config_puppeteer_chromium_revision - || require('./package.json').puppeteer.chromium_revision; - -const revisionInfo = browserFetcher.revisionInfo(revision); - -// Do nothing if the revision is already downloaded. -if (revisionInfo.local) { - generateProtocolTypesIfNecessary(false /* updated */); - return; -} - -// Override current environment proxy settings with npm configuration, if any. -const NPM_HTTPS_PROXY = process.env.npm_config_https_proxy || process.env.npm_config_proxy; -const NPM_HTTP_PROXY = process.env.npm_config_http_proxy || process.env.npm_config_proxy; -const NPM_NO_PROXY = process.env.npm_config_no_proxy; - -if (NPM_HTTPS_PROXY) - process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; -if (NPM_HTTP_PROXY) - process.env.HTTP_PROXY = NPM_HTTP_PROXY; -if (NPM_NO_PROXY) - process.env.NO_PROXY = NPM_NO_PROXY; +const supportedProducts = { + 'chrome': 'Chromium', + 'firefox': 'Firefox Nightly' +}; + +async function download() { + const downloadHost = process.env.PUPPETEER_DOWNLOAD_HOST || process.env.npm_config_puppeteer_download_host || process.env.npm_package_config_puppeteer_download_host; + const puppeteer = require('./index'); + const product = process.env.PUPPETEER_PRODUCT || process.env.npm_config_puppeteer_product || process.env.npm_package_config_puppeteer_product || 'chrome'; + const browserFetcher = puppeteer.createBrowserFetcher({ product, host: downloadHost }); + const revision = await getRevision(); + await fetchBinary(revision); + + function getRevision() { + if (product === 'chrome') { + return process.env.PUPPETEER_CHROMIUM_REVISION || process.env.npm_config_puppeteer_chromium_revision || process.env.npm_package_config_puppeteer_chromium_revision + || require('./package.json').puppeteer.chromium_revision; + } else if (product === 'firefox') { + puppeteer._preferredRevision = require('./package.json').puppeteer.firefox_revision; + return getFirefoxNightlyVersion(browserFetcher.host()).catch(error => { console.error(error); process.exit(1); }); + } else { + throw new Error(`Unsupported product ${product}`); + } + } -browserFetcher.download(revisionInfo.revision, onProgress) - .then(() => browserFetcher.localRevisions()) - .then(onSuccess) - .catch(onError); + function fetchBinary(revision) { + const revisionInfo = browserFetcher.revisionInfo(revision); + + // Do nothing if the revision is already downloaded. + if (revisionInfo.local) { + generateProtocolTypesIfNecessary(false /* updated */, product); + logPolitely(`${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.`); + return; + } + + // Override current environment proxy settings with npm configuration, if any. + const NPM_HTTPS_PROXY = process.env.npm_config_https_proxy || process.env.npm_config_proxy; + const NPM_HTTP_PROXY = process.env.npm_config_http_proxy || process.env.npm_config_proxy; + const NPM_NO_PROXY = process.env.npm_config_no_proxy; + + if (NPM_HTTPS_PROXY) + process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; + if (NPM_HTTP_PROXY) + process.env.HTTP_PROXY = NPM_HTTP_PROXY; + if (NPM_NO_PROXY) + process.env.NO_PROXY = NPM_NO_PROXY; + + /** + * @param {!Array} + * @return {!Promise} + */ + function onSuccess(localRevisions) { + logPolitely(`${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}`); + localRevisions = localRevisions.filter(revision => revision !== revisionInfo.revision); + const cleanupOldVersions = localRevisions.map(revision => browserFetcher.remove(revision)); + Promise.all([...cleanupOldVersions, generateProtocolTypesIfNecessary(true /* updated */, product)]); + } + + /** + * @param {!Error} error + */ + function onError(error) { + console.error(`ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`); + console.error(error); + process.exit(1); + } + + let progressBar = null; + let lastDownloadedBytes = 0; + function onProgress(downloadedBytes, totalBytes) { + if (!progressBar) { + const ProgressBar = require('progress'); + progressBar = new ProgressBar(`Downloading ${supportedProducts[product]} r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + }); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + } + + return browserFetcher.download(revisionInfo.revision, onProgress) + .then(() => browserFetcher.localRevisions()) + .then(onSuccess) + .catch(onError); + } -/** - * @param {!Array} - * @return {!Promise} - */ -function onSuccess(localRevisions) { - logPolitely('Chromium downloaded to ' + revisionInfo.folderPath); - localRevisions = localRevisions.filter(revision => revision !== revisionInfo.revision); - // Remove previous chromium revisions. - const cleanupOldVersions = localRevisions.map(revision => browserFetcher.remove(revision)); - return Promise.all([...cleanupOldVersions, generateProtocolTypesIfNecessary(true /* updated */)]); -} + function toMegabytes(bytes) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; + } -/** - * @param {!Error} error - */ -function onError(error) { - console.error(`ERROR: Failed to download Chromium r${revision}! Set "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" env variable to skip download.`); - console.error(error); - process.exit(1); -} + function generateProtocolTypesIfNecessary(updated, product) { + if (product !== 'chrome') + return; + const fs = require('fs'); + const path = require('path'); + if (!fs.existsSync(path.join(__dirname, 'utils', 'protocol-types-generator'))) + return; + if (!updated && fs.existsSync(path.join(__dirname, 'lib', 'protocol.d.ts'))) + return; + return require('./utils/protocol-types-generator'); + } -let progressBar = null; -let lastDownloadedBytes = 0; -function onProgress(downloadedBytes, totalBytes) { - if (!progressBar) { - const ProgressBar = require('progress'); - progressBar = new ProgressBar(`Downloading Chromium r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, + function getFirefoxNightlyVersion(host) { + const https = require('https'); + const promise = new Promise((resolve, reject) => { + let data = ''; + logPolitely(`Requesting latest Firefox Nightly version from ${host}`); + https.get(host + '/', r => { + if (r.statusCode >= 400) + return reject(new Error(`Got status code ${r.statusCode}`)); + r.on('data', chunk => { + data += chunk; + }); + r.on('end', parseVersion); + }).on('error', reject); + + function parseVersion() { + const regex = /firefox\-(?\d\d)\..*/gm; + let result = 0; + let match; + while ((match = regex.exec(data)) !== null) { + const version = parseInt(match.groups.version, 10); + if (version > result) + result = version; + } + if (result) + resolve(result.toString()); + else reject(new Error('Firefox version not found')); + } }); + return promise; } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); -} - -function toMegabytes(bytes) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - -function generateProtocolTypesIfNecessary(updated) { - const fs = require('fs'); - const path = require('path'); - if (!fs.existsSync(path.join(__dirname, 'utils', 'protocol-types-generator'))) - return; - if (!updated && fs.existsSync(path.join(__dirname, 'lib', 'protocol.d.ts'))) - return; - return require('./utils/protocol-types-generator'); } function logPolitely(toBeLogged) { @@ -129,3 +170,30 @@ function logPolitely(toBeLogged) { console.log(toBeLogged); } +if (process.env.PUPPETEER_SKIP_DOWNLOAD) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.'); + return; +} +if (process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD || process.env.npm_config_puppeteer_skip_download) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.'); + return; +} +if (process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD || process.env.npm_package_config_puppeteer_skip_download) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.'); + return; +} +if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.'); + return; +} +if (process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || process.env.npm_config_puppeteer_skip_chromium_download) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.'); + return; +} +if (process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || process.env.npm_package_config_puppeteer_skip_chromium_download) { + logPolitely('**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.'); + return; +} + +download(); + diff --git a/lib/BrowserFetcher.js b/lib/BrowserFetcher.js index aace9220591f4..f916fe73ae4e0 100644 --- a/lib/BrowserFetcher.js +++ b/lib/BrowserFetcher.js @@ -18,7 +18,9 @@ const os = require('os'); const fs = require('fs'); const path = require('path'); const util = require('util'); +const childProcess = require('child_process'); const extract = require('extract-zip'); +const debugFetcher = require('debug')(`puppeteer:fetcher`); const URL = require('url'); const {helper, assert} = require('./helper'); const removeRecursive = require('rimraf'); @@ -27,41 +29,64 @@ const ProxyAgent = require('https-proxy-agent'); // @ts-ignore const getProxyForUrl = require('proxy-from-env').getProxyForUrl; -const DEFAULT_DOWNLOAD_HOST = 'https://storage.googleapis.com'; - -const supportedPlatforms = ['mac', 'linux', 'win32', 'win64']; const downloadURLs = { - linux: '%s/chromium-browser-snapshots/Linux_x64/%d/%s.zip', - mac: '%s/chromium-browser-snapshots/Mac/%d/%s.zip', - win32: '%s/chromium-browser-snapshots/Win/%d/%s.zip', - win64: '%s/chromium-browser-snapshots/Win_x64/%d/%s.zip', + chrome: { + linux: '%s/chromium-browser-snapshots/Linux_x64/%d/%s.zip', + mac: '%s/chromium-browser-snapshots/Mac/%d/%s.zip', + win32: '%s/chromium-browser-snapshots/Win/%d/%s.zip', + win64: '%s/chromium-browser-snapshots/Win_x64/%d/%s.zip', + }, + firefox: { + linux: '%s/firefox-%s.0a1.en-US.%s-x86_64.tar.bz2', + mac: '%s/firefox-%s.0a1.en-US.%s.dmg', + win32: '%s/firefox-%s.0a1.en-US.%s.zip', + win64: '%s/firefox-%s.0a1.en-US.%s.zip', + }, +}; + +const browserConfig = { + chrome: { + host: 'https://storage.googleapis.com', + destination: '.local-chromium', + }, + firefox: { + host: 'https://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central', + destination: '.local-firefox', + } }; /** + * @param {string} product * @param {string} platform * @param {string} revision * @return {string} */ -function archiveName(platform, revision) { - if (platform === 'linux') - return 'chrome-linux'; - if (platform === 'mac') - return 'chrome-mac'; - if (platform === 'win32' || platform === 'win64') { - // Windows archive name changed at r591479. - return parseInt(revision, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; +function archiveName(product, platform, revision) { + if (product === 'chrome') { + if (platform === 'linux') + return 'chrome-linux'; + if (platform === 'mac') + return 'chrome-mac'; + if (platform === 'win32' || platform === 'win64') { + // Windows archive name changed at r591479. + return parseInt(revision, 10) > 591479 ? 'chrome-win' : 'chrome-win32'; + } + } else if (product === 'firefox') { + return platform; } return null; } /** + * @param {string} product * @param {string} platform * @param {string} host * @param {string} revision * @return {string} */ -function downloadURL(platform, host, revision) { - return util.format(downloadURLs[platform], host, revision, archiveName(platform, revision)); +function downloadURL(product, platform, host, revision) { + const url = util.format(downloadURLs[product][platform], host, revision, archiveName(product, platform, revision)); + return url; } const readdirAsync = helper.promisify(fs.readdir.bind(fs)); @@ -82,8 +107,10 @@ class BrowserFetcher { * @param {!BrowserFetcher.Options=} options */ constructor(projectRoot, options = {}) { - this._downloadsFolder = options.path || path.join(projectRoot, '.local-chromium'); - this._downloadHost = options.host || DEFAULT_DOWNLOAD_HOST; + this._product = (options.product || 'chrome').toLowerCase(); + assert(this._product === 'chrome' || this._product === 'firefox', `Unknown product: "${options.product}"`); + this._downloadsFolder = options.path || path.join(projectRoot, browserConfig[this._product].destination); + this._downloadHost = options.host || browserConfig[this._product].host; this._platform = options.platform || ''; if (!this._platform) { const platform = os.platform(); @@ -95,7 +122,7 @@ class BrowserFetcher { this._platform = os.arch() === 'x64' ? 'win64' : 'win32'; assert(this._platform, 'Unsupported platform: ' + os.platform()); } - assert(supportedPlatforms.includes(this._platform), 'Unsupported platform: ' + this._platform); + assert(downloadURLs[this._product][this._platform], 'Unsupported platform: ' + this._platform); } /** @@ -105,12 +132,26 @@ class BrowserFetcher { return this._platform; } + /** + * @return {string} + */ + product() { + return this._product; + } + + /** + * @return {string} + */ + host() { + return this._downloadHost; + } + /** * @param {string} revision * @return {!Promise} */ canDownload(revision) { - const url = downloadURL(this._platform, this._downloadHost, revision); + const url = downloadURL(this._product, this._platform, this._downloadHost, revision); let resolve; const promise = new Promise(x => resolve = x); const request = httpRequest(url, 'HEAD', response => { @@ -129,19 +170,20 @@ class BrowserFetcher { * @return {!Promise} */ async download(revision, progressCallback) { - const url = downloadURL(this._platform, this._downloadHost, revision); - const zipPath = path.join(this._downloadsFolder, `download-${this._platform}-${revision}.zip`); - const folderPath = this._getFolderPath(revision); - if (await existsAsync(folderPath)) + const url = downloadURL(this._product, this._platform, this._downloadHost, revision); + const fileName = url.split('/').pop(); + const archivePath = path.join(this._downloadsFolder, fileName); + const outputPath = this._getFolderPath(revision); + if (await existsAsync(outputPath)) return this.revisionInfo(revision); if (!(await existsAsync(this._downloadsFolder))) await mkdirAsync(this._downloadsFolder); try { - await downloadFile(url, zipPath, progressCallback); - await extractZip(zipPath, folderPath); + await downloadFile(url, archivePath, progressCallback); + await install(archivePath, outputPath); } finally { - if (await existsAsync(zipPath)) - await unlinkAsync(zipPath); + if (await existsAsync(archivePath)) + await unlinkAsync(archivePath); } const revisionInfo = this.revisionInfo(revision); if (revisionInfo) @@ -156,7 +198,7 @@ class BrowserFetcher { if (!await existsAsync(this._downloadsFolder)) return []; const fileNames = await readdirAsync(this._downloadsFolder); - return fileNames.map(fileName => parseFolderPath(fileName)).filter(entry => entry && entry.platform === this._platform).map(entry => entry.revision); + return fileNames.map(fileName => parseFolderPath(this._product, fileName)).filter(entry => entry && entry.platform === this._platform).map(entry => entry.revision); } /** @@ -175,17 +217,31 @@ class BrowserFetcher { revisionInfo(revision) { const folderPath = this._getFolderPath(revision); let executablePath = ''; - if (this._platform === 'mac') - executablePath = path.join(folderPath, archiveName(this._platform, revision), 'Chromium.app', 'Contents', 'MacOS', 'Chromium'); - else if (this._platform === 'linux') - executablePath = path.join(folderPath, archiveName(this._platform, revision), 'chrome'); - else if (this._platform === 'win32' || this._platform === 'win64') - executablePath = path.join(folderPath, archiveName(this._platform, revision), 'chrome.exe'); - else - throw new Error('Unsupported platform: ' + this._platform); - const url = downloadURL(this._platform, this._downloadHost, revision); + if (this._product === 'chrome') { + if (this._platform === 'mac') + executablePath = path.join(folderPath, archiveName(this._product, this._platform, revision), 'Chromium.app', 'Contents', 'MacOS', 'Chromium'); + else if (this._platform === 'linux') + executablePath = path.join(folderPath, archiveName(this._product, this._platform, revision), 'chrome'); + else if (this._platform === 'win32' || this._platform === 'win64') + executablePath = path.join(folderPath, archiveName(this._product, this._platform, revision), 'chrome.exe'); + else + throw new Error('Unsupported platform: ' + this._platform); + } else if (this._product === 'firefox') { + if (this._platform === 'mac') + executablePath = path.join(folderPath, 'Firefox Nightly.app', 'Contents', 'MacOS', 'firefox'); + else if (this._platform === 'linux') + executablePath = path.join(folderPath, 'firefox', 'firefox'); + else if (this._platform === 'win32' || this._platform === 'win64') + executablePath = path.join(folderPath, 'firefox', 'firefox.exe'); + else + throw new Error('Unsupported platform: ' + this._platform); + } else { + throw new Error('Unsupported product: ' + this._product); + } + const url = downloadURL(this._product, this._platform, this._downloadHost, revision); const local = fs.existsSync(folderPath); - return {revision, executablePath, folderPath, local, url}; + debugFetcher({revision, executablePath, folderPath, local, url, product: this._product}); + return {revision, executablePath, folderPath, local, url, product: this._product}; } /** @@ -201,17 +257,17 @@ module.exports = BrowserFetcher; /** * @param {string} folderPath - * @return {?{platform: string, revision: string}} + * @return {?{product: string, platform: string, revision: string}} */ -function parseFolderPath(folderPath) { +function parseFolderPath(product, folderPath) { const name = path.basename(folderPath); const splits = name.split('-'); if (splits.length !== 2) return null; const [platform, revision] = splits; - if (!supportedPlatforms.includes(platform)) + if (!downloadURLs[product][platform]) return null; - return {platform, revision}; + return {product, platform, revision}; } /** @@ -221,6 +277,7 @@ function parseFolderPath(folderPath) { * @return {!Promise} */ function downloadFile(url, destinationPath, progressCallback) { + debugFetcher(`Downloading binary from ${url}`); let fulfill, reject; let downloadedBytes = 0; let totalBytes = 0; @@ -252,6 +309,26 @@ function downloadFile(url, destinationPath, progressCallback) { } } + +/** + * Install from a zip, tar.bz2 or dmg file. + * + * @param {string} archivePath + * @param {string} folderPath + * @return {!Promise} + */ +function install(archivePath, folderPath) { + debugFetcher(`Installing ${archivePath} to ${folderPath}`); + if (archivePath.endsWith('.zip')) + return extractZip(archivePath, folderPath); + else if (archivePath.endsWith('.tar.bz2')) + return extractTar(archivePath, folderPath); + else if (archivePath.endsWith('.dmg')) + return mkdirAsync(folderPath).then(() => installDMG(archivePath, folderPath)); + else + throw new Error(`Unsupported archive format: ${archivePath}`); +} + /** * @param {string} zipPath * @param {string} folderPath @@ -266,6 +343,74 @@ function extractZip(zipPath, folderPath) { })); } +/** + * @param {string} tarPath + * @param {string} folderPath + * @return {!Promise} + */ +function extractTar(tarPath, folderPath) { + const tar = require('tar-fs'); + // @ts-ignore + const bzip = require('unbzip2-stream'); + return new Promise((fulfill, reject) => { + const tarStream = tar.extract(folderPath); + tarStream.on('error', reject); + tarStream.on('finish', fulfill); + const readStream = fs.createReadStream(tarPath); + readStream.on('data', () => { process.stdout.write('\rExtracting...'); }); + readStream.pipe(bzip()).pipe(tarStream); + }); +} + +/** + * Install *.app directory from dmg file + * + * @param {string} dmgPath + * @param {string} folderPath + * @return {!Promise} + */ +function installDMG(dmgPath, folderPath) { + let mountPath; + + function mountAndCopy(fulfill, reject) { + const mountCommand = `hdiutil attach -nobrowse -noautoopen "${dmgPath}"`; + childProcess.exec(mountCommand, (err, stdout, stderr) => { + if (err) + return reject(err); + const volumes = stdout.match(/\/Volumes\/(.*)/m); + if (!volumes) + return reject(new Error(`Could not find volume path in ${stdout}`)); + mountPath = volumes[0]; + readdirAsync(mountPath).then(fileNames => { + const appName = fileNames.filter(item => typeof item === 'string' && item.endsWith('.app'))[0]; + if (!appName) + return reject(new Error(`Cannot find app in ${mountPath}`)); + const copyPath = path.join(mountPath, appName); + debugFetcher(`Copying ${copyPath} to ${folderPath}`); + childProcess.exec(`cp -R "${copyPath}" "${folderPath}"`, (err, stdout) => { + if (err) + reject(err); + else + fulfill(); + }); + }).catch(reject); + }); + } + + function unmount() { + if (!mountPath) + return; + const unmountCommand = `hdiutil detach "${mountPath}" -quiet`; + debugFetcher(`Unmounting ${mountPath}`); + childProcess.exec(unmountCommand, err => { + if (err) + console.error(`Error unmounting dmg: ${err}`); + }); + } + + return new Promise(mountAndCopy).catch(err => { console.error(err); }).finally(unmount); +} + function httpRequest(url, method, response) { /** @type {Object} */ let options = URL.parse(url); @@ -306,6 +451,7 @@ function httpRequest(url, method, response) { /** * @typedef {Object} BrowserFetcher.Options * @property {string=} platform + * @property {string=} product * @property {string=} path * @property {string=} host */ @@ -317,4 +463,5 @@ function httpRequest(url, method, response) { * @property {string} url * @property {boolean} local * @property {string} revision + * @property {string} product */ diff --git a/lib/Launcher.js b/lib/Launcher.js index 2c4f238f1d882..5d5651b72aaf5 100644 --- a/lib/Launcher.js +++ b/lib/Launcher.js @@ -411,15 +411,23 @@ class FirefoxLauncher { firefoxArguments.push(temporaryUserDataDir); } - let executable = executablePath; + // replace 'latest' placeholder with actual downloaded revision + if (this._preferredRevision === 'latest') { + const browserFetcher = new BrowserFetcher(this._projectRoot, {product: this.product}); + const localRevisions = await browserFetcher.localRevisions(); + if (localRevisions[0]) + this._preferredRevision = localRevisions[0]; + } + + let firefoxExecutable = executablePath; if (!executablePath) { const {missingText, executablePath} = resolveExecutablePath(this); if (missingText) throw new Error(missingText); - executable = executablePath; + firefoxExecutable = executablePath; } - const runner = new BrowserRunner(executable, firefoxArguments, temporaryUserDataDir); + const runner = new BrowserRunner(firefoxExecutable, firefoxArguments, temporaryUserDataDir); runner.start({handleSIGHUP, handleSIGTERM, handleSIGINT, dumpio, env, pipe}); try { @@ -469,11 +477,7 @@ class FirefoxLauncher { * @return {string} */ executablePath() { - const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || process.env.npm_config_puppeteer_executable_path || process.env.npm_package_config_puppeteer_executable_path; - // TODO get resolveExecutablePath working for Firefox - if (!executablePath) - throw new Error('Please set PUPPETEER_EXECUTABLE_PATH to a Firefox binary.'); - return executablePath; + return resolveExecutablePath(this).executablePath; } /** @@ -832,8 +836,8 @@ function resolveExecutablePath(launcher) { return { executablePath, missingText }; } } - const browserFetcher = new BrowserFetcher(launcher._projectRoot); - if (!launcher._isPuppeteerCore) { + const browserFetcher = new BrowserFetcher(launcher._projectRoot, {product: launcher.product}); + if (!launcher._isPuppeteerCore && launcher.product === 'chrome') { const revision = process.env['PUPPETEER_CHROMIUM_REVISION']; if (revision) { const revisionInfo = browserFetcher.revisionInfo(revision); @@ -842,7 +846,7 @@ function resolveExecutablePath(launcher) { } } const revisionInfo = browserFetcher.revisionInfo(launcher._preferredRevision); - const missingText = !revisionInfo.local ? `Browser is not downloaded. Run "npm install" or "yarn install"` : null; + const missingText = !revisionInfo.local ? `Could not find browser revision ${launcher._preferredRevision}. Run "npm install" or "yarn install" to download a browser binary.` : null; return {executablePath: revisionInfo.executablePath, missingText}; } diff --git a/lib/Puppeteer.js b/lib/Puppeteer.js index f54ef62a6ad5c..7b12ae9b4f6c8 100644 --- a/lib/Puppeteer.js +++ b/lib/Puppeteer.js @@ -34,9 +34,8 @@ module.exports = class { * @param {!(Launcher.LaunchOptions & Launcher.ChromeArgOptions & Launcher.BrowserOptions & {product?: string, extraPrefsFirefox?: !object})=} options * @return {!Promise} */ - launch(options) { - if (!this._productName && options) - this._productName = options.product; + launch(options = {}) { + this._productName = options.product; return this._launcher.launch(options); } @@ -59,10 +58,20 @@ module.exports = class { * @return {!Puppeteer.ProductLauncher} */ get _launcher() { - if (!this._lazyLauncher) + if (!this._lazyLauncher || this._lazyLauncher.product !== this._productName) { + // @ts-ignore + const packageJson = require('../package.json'); + switch (this._productName) { + case 'firefox': + this._preferredRevision = packageJson.puppeteer.firefox_revision; + break; + case 'chrome': + default: + this._preferredRevision = packageJson.puppeteer.chromium_revision; + } this._lazyLauncher = Launcher(this._projectRoot, this._preferredRevision, this._isPuppeteerCore, this._productName); + } return this._lazyLauncher; - } /** diff --git a/package.json b/package.json index 3f44c6e2b79b9..1decf3e7ec0ef 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "node": ">=10.18.1" }, "puppeteer": { - "chromium_revision": "722234" + "chromium_revision": "722234", + "firefox_revision": "latest" }, "scripts": { "unit": "node test/test.js", @@ -38,6 +39,8 @@ "progress": "^2.0.1", "proxy-from-env": "^1.0.0", "rimraf": "^2.6.1", + "tar-fs": "^2.0.0", + "unbzip2-stream": "^1.3.3", "ws": "^6.1.0" }, "devDependencies": { @@ -46,6 +49,7 @@ "@types/mime": "^2.0.0", "@types/node": "^10.17.14", "@types/rimraf": "^2.0.2", + "@types/tar-fs": "^1.16.2", "@types/ws": "^6.0.1", "commonmark": "^0.28.1", "cross-env": "^5.0.5", diff --git a/test/assets/firefox-75.0a1.en-US.linux-x86_64.tar.bz2 b/test/assets/firefox-75.0a1.en-US.linux-x86_64.tar.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..be6d1880276d5efa4b65beaf1c364585a89356f9 GIT binary patch literal 211 zcmV;^04)DPT4*^jL0KkKS^PvgW&i-Ze~{9d0DwRM|9~?kvS4pwp6~*+gqZ)>q7y=pwfNe)&bw4*ux`+}2JqZA%V*vC( zFhVj4{HSRG2#P;0j7;1K7p{Qj48j}WqXH8YymE-^g+#`9Uu1Pelwwb676jyCL_#V+ zpf^utwby0I-VFJ#T<1xWj==bG3N0bVqa5Mjoel@VQcODrBI8%l { + it('should download and extract chrome linux binary', async({server}) => { const downloadsFolder = await mkdtempAsync(TMP_FOLDER); const browserFetcher = puppeteer.createBrowserFetcher({ platform: 'linux', path: downloadsFolder, host: server.PREFIX }); - let revisionInfo = browserFetcher.revisionInfo('123456'); + const expectedRevision = '123456'; + let revisionInfo = browserFetcher.revisionInfo(expectedRevision); server.setRoute(revisionInfo.url.substring(server.PREFIX.length), (req, res) => { server.serveFile(req, res, '/chromium-linux.zip'); }); expect(revisionInfo.local).toBe(false); expect(browserFetcher.platform()).toBe('linux'); + expect(browserFetcher.product()).toBe('chrome'); + expect(!!browserFetcher.host()).toBe(true); expect(await browserFetcher.canDownload('100000')).toBe(false); - expect(await browserFetcher.canDownload('123456')).toBe(true); + expect(await browserFetcher.canDownload(expectedRevision)).toBe(true); - revisionInfo = await browserFetcher.download('123456'); + revisionInfo = await browserFetcher.download(expectedRevision); expect(revisionInfo.local).toBe(true); expect(await readFileAsync(revisionInfo.executablePath, 'utf8')).toBe('LINUX BINARY\n'); - const expectedPermissions = os.platform() === 'win32' ? 0666 : 0755; - expect((await statAsync(revisionInfo.executablePath)).mode & 0777).toBe(expectedPermissions); - expect(await browserFetcher.localRevisions()).toEqual(['123456']); - await browserFetcher.remove('123456'); + const expectedPermissions = os.platform() === 'win32' ? 0o666 : 0o755; + expect((await statAsync(revisionInfo.executablePath)).mode & 0o777).toBe(expectedPermissions); + expect(await browserFetcher.localRevisions()).toEqual([expectedRevision]); + await browserFetcher.remove(expectedRevision); + expect(await browserFetcher.localRevisions()).toEqual([]); + await rmAsync(downloadsFolder); + }); + }); + describe('BrowserFetcher', function() { + it('should download and extract firefox linux binary', async({server}) => { + const downloadsFolder = await mkdtempAsync(TMP_FOLDER); + const browserFetcher = puppeteer.createBrowserFetcher({ + platform: 'linux', + path: downloadsFolder, + host: server.PREFIX, + product: 'firefox', + }); + const expectedVersion = '75'; + let revisionInfo = browserFetcher.revisionInfo(expectedVersion); + server.setRoute(revisionInfo.url.substring(server.PREFIX.length), (req, res) => { + server.serveFile(req, res, `/firefox-${expectedVersion}.0a1.en-US.linux-x86_64.tar.bz2`); + }); + + expect(revisionInfo.local).toBe(false); + expect(browserFetcher.platform()).toBe('linux'); + expect(browserFetcher.product()).toBe('firefox'); + expect(await browserFetcher.canDownload('100000')).toBe(false); + expect(await browserFetcher.canDownload(expectedVersion)).toBe(true); + + revisionInfo = await browserFetcher.download(expectedVersion); + expect(revisionInfo.local).toBe(true); + expect(await readFileAsync(revisionInfo.executablePath, 'utf8')).toBe('FIREFOX LINUX BINARY\n'); + const expectedPermissions = os.platform() === 'win32' ? 0o666 : 0o755; + expect((await statAsync(revisionInfo.executablePath)).mode & 0o777).toBe(expectedPermissions); + expect(await browserFetcher.localRevisions()).toEqual([expectedVersion]); + await browserFetcher.remove(expectedVersion); expect(await browserFetcher.localRevisions()).toEqual([]); await rmAsync(downloadsFolder); });