From 1f5e333f0059bf3f7d9bba7e5376ebddff591c41 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Mon, 29 Jun 2020 16:13:24 +0100 Subject: [PATCH] chore: Don't store revisions in `package.json` (#6109) * chore: Don't store revisions in `package.json` It's quite messy to have to require the `package.json` file in multiple places purely to find out what revision of a given browser we want to use. We can also achieve better type safety by placing it in an actual source file. This commit makes that change and also tidies up our reliance on `package.json` within the source code generally; we now only use it to find the location of the Puppeteer root such that we know where to install downloaded browsers to. To avoid using `package.json` to parse the name of the module, we also now explicitly have an entry point for the Puppeteer module and the Puppeter Core module. This will make it easier in the future to ship less code as part of core (e.g. core never needs to download a browser, so why ship that code?). Core can also then not have any revisions based info contained in it. The test install script has also been updated to ensure that puppeteer-core can be installed correctly too. Finally, the `install` script has been moved to TypeScript for nicer typechecking and safety. The functionality of it has not changed. --- .gitignore | 1 + .travis.yml | 10 +- README.md | 4 +- cjs-entry-core.js | 29 +++ install.js | 255 ++++--------------- new-docs/puppeteer.puppeteer._projectroot.md | 11 - new-docs/puppeteer.puppeteer.md | 1 - package.json | 10 +- scripts/test-install.sh | 17 ++ src/common/Puppeteer.ts | 10 +- src/index-core.ts | 20 ++ src/index.ts | 17 +- src/initialize.ts | 25 +- src/install.ts | 176 +++++++++++++ src/revisions.ts | 25 ++ utils/check_availability.js | 7 +- utils/prepare_puppeteer_core.js | 1 + 17 files changed, 349 insertions(+), 270 deletions(-) create mode 100644 cjs-entry-core.js delete mode 100644 new-docs/puppeteer.puppeteer._projectroot.md create mode 100644 src/index-core.ts create mode 100644 src/install.ts create mode 100644 src/revisions.ts diff --git a/.gitignore b/.gitignore index 89804cb891757..c5f0b19c647cc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ yarn.lock test/coverage.json temp/ dependency-chart.png +puppeteer-core-*.tgz diff --git a/.travis.yml b/.travis.yml index 467f88577b8c1..8da18b9a8579a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,11 +64,19 @@ jobs: - CHROMIUM=true script: - npm run compare-protocol-d-ts - - npm run test-install - npm run lint - npm run test-doclint - npm run ensure-new-docs-up-to-date + # This bot runs separately as it changes package.json to test puppeteer-core + # and we don't want that leaking into other bots and causing issues. + - node_js: "10.19.0" + name: 'Test bundling and install of packages' + env: + - CHROMIUM=true + script: + - npm run test-install + # Runs unit tests on Linux + Firefox - node_js: "10.19.0" name: 'Unit tests: Linux/Firefox' diff --git a/README.md b/README.md index 7bd432809f21d..e36a82a5b6deb 100644 --- a/README.md +++ b/README.md @@ -380,12 +380,12 @@ npm install puppeteer-core@chrome-71 #### Q: Which Chromium version does Puppeteer use? -Look for `chromium_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section. +Look for the `chromium` entry in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section. #### Q: Which Firefox version does Puppeteer use? -Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json) is `latest` -- Puppeteer isn't tied to a particular Firefox version. +Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox` in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts) is `latest` -- Puppeteer isn't tied to a particular Firefox version. To fetch Firefox Nightly as part of Puppeteer installation: diff --git a/cjs-entry-core.js b/cjs-entry-core.js new file mode 100644 index 0000000000000..efcdb39027f7e --- /dev/null +++ b/cjs-entry-core.js @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * We use `export default puppeteer` in `src/index.ts` to expose the library But + * TypeScript in CJS mode compiles that to `exports.default = `. This means that + * our CJS Node users would have to use `require('puppeteer').default` which + * isn't very nice. + * + * So instead we expose this file as our entry point. This requires the compiled + * Puppeteer output and re-exports the `default` export via `module.exports.` + * This means that we can publish to CJS and ESM whilst maintaining the expected + * import behaviour for CJS and ESM users. + */ +const puppeteerExport = require('./lib/cjs/index-core'); +module.exports = puppeteerExport.default; diff --git a/install.js b/install.js index cf56da410ea48..402a72e0d2f60 100644 --- a/install.js +++ b/install.js @@ -25,221 +25,62 @@ */ const compileTypeScriptIfRequired = require('./typescript-if-required'); -const os = require('os'); - -const firefoxVersions = - 'https://product-details.mozilla.org/1.0/firefox_versions.json'; -const supportedProducts = { - chrome: 'Chromium', - firefox: 'Firefox Nightly', -}; async function download() { await compileTypeScriptIfRequired(); - - const downloadHost = - process.env.PUPPETEER_DOWNLOAD_HOST || - process.env.npm_config_puppeteer_download_host || - process.env.npm_package_config_puppeteer_download_host; - const puppeteer = require('.'); - const product = - process.env.PUPPETEER_PRODUCT || - process.env.npm_config_puppeteer_product || - process.env.npm_package_config_puppeteer_product || - 'chrome'; - const browserFetcher = puppeteer.createBrowserFetcher({ - product, - host: downloadHost, - }); - const revision = await getRevision(); - await fetchBinary(revision); - - function getRevision() { - if (product === 'chrome') { - return ( - process.env.PUPPETEER_CHROMIUM_REVISION || - process.env.npm_config_puppeteer_chromium_revision || - process.env.npm_package_config_puppeteer_chromium_revision || - require('./package.json').puppeteer.chromium_revision - ); - } else if (product === 'firefox') { - puppeteer._preferredRevision = require('./package.json').puppeteer.firefox_revision; - return getFirefoxNightlyVersion(browserFetcher.host()).catch((error) => { - console.error(error); - process.exit(1); - }); - } else { - throw new Error(`Unsupported product ${product}`); - } + // need to ensure TS is compiled before loading the installer + const { downloadBrowser, logPolitely } = require('./lib/cjs/install'); + + if (process.env.PUPPETEER_SKIP_DOWNLOAD) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.' + ); + return; } - - function fetchBinary(revision) { - const revisionInfo = browserFetcher.revisionInfo(revision); - - // Do nothing if the revision is already downloaded. - if (revisionInfo.local) { - logPolitely( - `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` - ); - return; - } - - // Override current environment proxy settings with npm configuration, if any. - const NPM_HTTPS_PROXY = - process.env.npm_config_https_proxy || process.env.npm_config_proxy; - const NPM_HTTP_PROXY = - process.env.npm_config_http_proxy || process.env.npm_config_proxy; - const NPM_NO_PROXY = process.env.npm_config_no_proxy; - - if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; - if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY; - if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY; - - /** - * @param {!Array} - * @returns {!Promise} - */ - function onSuccess(localRevisions) { - if (os.arch() !== 'arm64') { - logPolitely( - `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` - ); - } - localRevisions = localRevisions.filter( - (revision) => revision !== revisionInfo.revision - ); - const cleanupOldVersions = localRevisions.map((revision) => - browserFetcher.remove(revision) - ); - Promise.all([...cleanupOldVersions]); - } - - /** - * @param {!Error} error - */ - function onError(error) { - console.error( - `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` - ); - console.error(error); - process.exit(1); - } - - let progressBar = null; - let lastDownloadedBytes = 0; - function onProgress(downloadedBytes, totalBytes) { - if (!progressBar) { - const ProgressBar = require('progress'); - progressBar = new ProgressBar( - `Downloading ${ - supportedProducts[product] - } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - } - - return browserFetcher - .download(revisionInfo.revision, onProgress) - .then(() => browserFetcher.localRevisions()) - .then(onSuccess) - .catch(onError); + if ( + process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD || + process.env.npm_config_puppeteer_skip_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.' + ); + return; } - - function toMegabytes(bytes) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; + if ( + process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD || + process.env.npm_package_config_puppeteer_skip_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.' + ); + return; } - - function getFirefoxNightlyVersion(host) { - const https = require('https'); - const promise = new Promise((resolve, reject) => { - let data = ''; - logPolitely(`Requesting latest Firefox Nightly version from ${host}`); - https - .get(firefoxVersions, (r) => { - if (r.statusCode >= 400) - return reject(new Error(`Got status code ${r.statusCode}`)); - r.on('data', (chunk) => { - data += chunk; - }); - r.on('end', () => { - try { - const versions = JSON.parse(data); - return resolve(versions.FIREFOX_NIGHTLY); - } catch { - return reject(new Error('Firefox version not found')); - } - }); - }) - .on('error', reject); - }); - return promise; + if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.' + ); + return; + } + if ( + process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || + process.env.npm_config_puppeteer_skip_chromium_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.' + ); + return; + } + if ( + process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || + process.env.npm_package_config_puppeteer_skip_chromium_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.' + ); + return; } -} - -function logPolitely(toBeLogged) { - const logLevel = process.env.npm_config_loglevel; - const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; - - if (!logLevelDisplay) console.log(toBeLogged); -} -if (process.env.PUPPETEER_SKIP_DOWNLOAD) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.' - ); - return; -} -if ( - process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD || - process.env.npm_config_puppeteer_skip_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.' - ); - return; -} -if ( - process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD || - process.env.npm_package_config_puppeteer_skip_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.' - ); - return; -} -if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.' - ); - return; -} -if ( - process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || - process.env.npm_config_puppeteer_skip_chromium_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.' - ); - return; -} -if ( - process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || - process.env.npm_package_config_puppeteer_skip_chromium_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.' - ); - return; + downloadBrowser(); } download(); diff --git a/new-docs/puppeteer.puppeteer._projectroot.md b/new-docs/puppeteer.puppeteer._projectroot.md deleted file mode 100644 index 215226cc8552f..0000000000000 --- a/new-docs/puppeteer.puppeteer._projectroot.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [puppeteer](./puppeteer.md) > [Puppeteer](./puppeteer.puppeteer.md) > [\_projectRoot](./puppeteer.puppeteer._projectroot.md) - -## Puppeteer.\_projectRoot property - -Signature: - -```typescript -_projectRoot: string; -``` diff --git a/new-docs/puppeteer.puppeteer.md b/new-docs/puppeteer.puppeteer.md index f27df977800aa..e96e126c0bb3d 100644 --- a/new-docs/puppeteer.puppeteer.md +++ b/new-docs/puppeteer.puppeteer.md @@ -42,7 +42,6 @@ const puppeteer = require('puppeteer'); | [\_isPuppeteerCore](./puppeteer.puppeteer._ispuppeteercore.md) | | boolean | | | [\_lazyLauncher](./puppeteer.puppeteer._lazylauncher.md) | | ProductLauncher | | | [\_preferredRevision](./puppeteer.puppeteer._preferredrevision.md) | | string | | -| [\_projectRoot](./puppeteer.puppeteer._projectroot.md) | | string | | | [devices](./puppeteer.puppeteer.devices.md) | | DevicesMap | | | [errors](./puppeteer.puppeteer.errors.md) | | [PuppeteerErrors](./puppeteer.puppeteererrors.md) | | | [product](./puppeteer.puppeteer.product.md) | | string | | diff --git a/package.json b/package.json index 565425af2ee39..be0a4dda42462 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,6 @@ "engines": { "node": ">=10.18.1" }, - "puppeteer": { - "chromium_revision": "756035", - "firefox_revision": "latest" - }, "scripts": { "unit": "tsc --version && mocha --config mocha-config/puppeteer-unit-tests.js", "unit-with-coverage": "cross-env COVERAGE=1 npm run unit", @@ -41,10 +37,10 @@ }, "files": [ "lib/", - "index.js", "install.js", "typescript-if-required.js", - "cjs-entry.js" + "cjs-entry.js", + "cjs-entry-core.js" ], "author": "The Chromium Authors", "license": "Apache-2.0", @@ -54,9 +50,9 @@ "https-proxy-agent": "^4.0.0", "mime": "^2.0.3", "mitt": "^2.0.1", + "pkg-dir": "^4.2.0", "progress": "^2.0.1", "proxy-from-env": "^1.0.0", - "read-pkg-up": "^7.0.1", "rimraf": "^3.0.2", "tar-fs": "^2.0.0", "unbzip2-stream": "^1.3.3", diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 2782d2acff224..76703abcb492f 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh set -e +ROOTDIR="$(pwd)" # Pack the module into a tarball npm pack tarball="$(realpath puppeteer-*.tgz)" @@ -22,3 +23,19 @@ PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" rm "${tarball}" ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-79.0a1/firefox/firefox + +# Again for puppeteer-core +cd $ROOTDIR +node ./utils/prepare_puppeteer_core.js +npm pack +tarball="$(realpath puppeteer-core-*.tgz)" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +# Check we can install from the tarball. +# This emulates installing from npm and ensures that: +# 1. we publish the right files in the `files` list from package.json +# 2. The install script works and correctly exits without errors +# 3. Requiring Puppeteer Core from Node works. +npm install --loglevel silent "${tarball}" +node --eval="require('puppeteer-core')" + diff --git a/src/common/Puppeteer.ts b/src/common/Puppeteer.ts index 9cb299a20d76c..b43beb973975d 100644 --- a/src/common/Puppeteer.ts +++ b/src/common/Puppeteer.ts @@ -23,8 +23,6 @@ import { ProductLauncher } from '../node/Launcher'; import { BrowserFetcher, BrowserFetcherOptions } from '../node/BrowserFetcher'; import { puppeteerErrors, PuppeteerErrors } from './Errors'; import { ConnectionTransport } from './ConnectionTransport'; -import readPkgUp from 'read-pkg-up'; - import { devicesMap } from './DeviceDescriptors'; import { DevicesMap } from './DeviceDescriptors'; import { Browser } from './Browser'; @@ -35,6 +33,7 @@ import { clearQueryHandlers, QueryHandler, } from './QueryHandler'; +import { PUPPETEER_REVISIONS } from '../revisions'; /** * The main Puppeteer class @@ -58,7 +57,7 @@ import { * @public */ export class Puppeteer { - _projectRoot: string; + private _projectRoot: string; _preferredRevision: string; _isPuppeteerCore: boolean; _changedProduct = false; @@ -173,14 +172,13 @@ export class Puppeteer { this._lazyLauncher.product !== this._productName || this._changedProduct ) { - const { packageJson } = readPkgUp.sync({ cwd: __dirname }); switch (this._productName) { case 'firefox': - this._preferredRevision = packageJson.puppeteer.firefox_revision; + this._preferredRevision = PUPPETEER_REVISIONS.firefox; break; case 'chrome': default: - this._preferredRevision = packageJson.puppeteer.chromium_revision; + this._preferredRevision = PUPPETEER_REVISIONS.chromium; } this._changedProduct = false; this._lazyLauncher = Launcher( diff --git a/src/index-core.ts b/src/index-core.ts new file mode 100644 index 0000000000000..692279f1273f9 --- /dev/null +++ b/src/index-core.ts @@ -0,0 +1,20 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { initializePuppeteer } from './initialize'; + +const puppeteer = initializePuppeteer('puppeteer-core'); +export default puppeteer; diff --git a/src/index.ts b/src/index.ts index c395a7690ae14..bcabc49357841 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,20 +14,7 @@ * limitations under the License. */ -import { initializePuppeteer, InitOptions } from './initialize'; -import * as path from 'path'; -import readPkgUp from 'read-pkg-up'; - -const packageJsonResult = readPkgUp.sync({ - cwd: __dirname, -}); -const packageJson = packageJsonResult.packageJson as unknown; - -const rootDir = path.dirname(packageJsonResult.path); - -const puppeteer = initializePuppeteer({ - packageJson: packageJson as InitOptions['packageJson'], - rootDirectory: rootDir, -}); +import { initializePuppeteer } from './initialize'; +const puppeteer = initializePuppeteer('puppeteer'); export default puppeteer; diff --git a/src/initialize.ts b/src/initialize.ts index 4cf16ecf4b798..429379c844c4c 100644 --- a/src/initialize.ts +++ b/src/initialize.ts @@ -21,28 +21,19 @@ const api = require('./api'); import { helper } from './common/helper'; import { Puppeteer } from './common/Puppeteer'; +import { PUPPETEER_REVISIONS } from './revisions'; +import pkgDir from 'pkg-dir'; -export interface InitOptions { - packageJson: { - puppeteer: { - chromium_revision: string; - firefox_revision: string; - }; - name: string; - }; - rootDirectory: string; -} - -export const initializePuppeteer = (options: InitOptions): Puppeteer => { - const { packageJson, rootDirectory } = options; +export const initializePuppeteer = (packageName: string): Puppeteer => { + const puppeteerRootDirectory = pkgDir.sync(__dirname); for (const className in api) { if (typeof api[className] === 'function') helper.installAsyncStackHooks(api[className]); } - let preferredRevision = packageJson.puppeteer.chromium_revision; - const isPuppeteerCore = packageJson.name === 'puppeteer-core'; + let preferredRevision = PUPPETEER_REVISIONS.chromium; + const isPuppeteerCore = packageName === 'puppeteer-core'; // puppeteer-core ignores environment variables const product = isPuppeteerCore ? undefined @@ -50,10 +41,10 @@ export const initializePuppeteer = (options: InitOptions): Puppeteer => { process.env.npm_config_puppeteer_product || process.env.npm_package_config_puppeteer_product; if (!isPuppeteerCore && product === 'firefox') - preferredRevision = packageJson.puppeteer.firefox_revision; + preferredRevision = PUPPETEER_REVISIONS.firefox; const puppeteer = new Puppeteer( - rootDirectory, + puppeteerRootDirectory, preferredRevision, isPuppeteerCore, product diff --git a/src/install.ts b/src/install.ts new file mode 100644 index 0000000000000..57872e93ccaa9 --- /dev/null +++ b/src/install.ts @@ -0,0 +1,176 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import https from 'https'; +import ProgressBar from 'progress'; +import puppeteer from './index'; +import { PUPPETEER_REVISIONS } from './revisions'; + +const firefoxVersions = + 'https://product-details.mozilla.org/1.0/firefox_versions.json'; + +const supportedProducts = { + chrome: 'Chromium', + firefox: 'Firefox Nightly', +} as const; + +export async function downloadBrowser() { + const downloadHost = + process.env.PUPPETEER_DOWNLOAD_HOST || + process.env.npm_config_puppeteer_download_host || + process.env.npm_package_config_puppeteer_download_host; + const product = + process.env.PUPPETEER_PRODUCT || + process.env.npm_config_puppeteer_product || + process.env.npm_package_config_puppeteer_product || + 'chrome'; + const browserFetcher = puppeteer.createBrowserFetcher({ + product, + host: downloadHost, + }); + const revision = await getRevision(); + await fetchBinary(revision); + + function getRevision() { + if (product === 'chrome') { + return ( + process.env.PUPPETEER_CHROMIUM_REVISION || + process.env.npm_config_puppeteer_chromium_revision || + PUPPETEER_REVISIONS.chromium + ); + } else if (product === 'firefox') { + puppeteer._preferredRevision = PUPPETEER_REVISIONS.firefox; + return getFirefoxNightlyVersion(browserFetcher.host()).catch((error) => { + console.error(error); + process.exit(1); + }); + } else { + throw new Error(`Unsupported product ${product}`); + } + } + + function fetchBinary(revision) { + const revisionInfo = browserFetcher.revisionInfo(revision); + + // Do nothing if the revision is already downloaded. + if (revisionInfo.local) { + logPolitely( + `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` + ); + return; + } + + // Override current environment proxy settings with npm configuration, if any. + const NPM_HTTPS_PROXY = + process.env.npm_config_https_proxy || process.env.npm_config_proxy; + const NPM_HTTP_PROXY = + process.env.npm_config_http_proxy || process.env.npm_config_proxy; + const NPM_NO_PROXY = process.env.npm_config_no_proxy; + + if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; + if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY; + if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY; + + function onSuccess(localRevisions: string[]): void { + if (os.arch() !== 'arm64') { + logPolitely( + `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` + ); + } + localRevisions = localRevisions.filter( + (revision) => revision !== revisionInfo.revision + ); + const cleanupOldVersions = localRevisions.map((revision) => + browserFetcher.remove(revision) + ); + Promise.all([...cleanupOldVersions]); + } + + function onError(error: Error) { + console.error( + `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` + ); + console.error(error); + process.exit(1); + } + + let progressBar = null; + let lastDownloadedBytes = 0; + function onProgress(downloadedBytes, totalBytes) { + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${ + supportedProducts[product] + } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + } + + return browserFetcher + .download(revisionInfo.revision, onProgress) + .then(() => browserFetcher.localRevisions()) + .then(onSuccess) + .catch(onError); + } + + function toMegabytes(bytes) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; + } + + function getFirefoxNightlyVersion(host) { + const promise = new Promise((resolve, reject) => { + let data = ''; + logPolitely(`Requesting latest Firefox Nightly version from ${host}`); + https + .get(firefoxVersions, (r) => { + if (r.statusCode >= 400) + return reject(new Error(`Got status code ${r.statusCode}`)); + r.on('data', (chunk) => { + data += chunk; + }); + r.on('end', () => { + try { + const versions = JSON.parse(data); + return resolve(versions.FIREFOX_NIGHTLY); + } catch { + return reject(new Error('Firefox version not found')); + } + }); + }) + .on('error', reject); + }); + return promise; + } +} + +export function logPolitely(toBeLogged) { + const logLevel = process.env.npm_config_loglevel; + const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; + + // eslint-disable-next-line no-console + if (!logLevelDisplay) console.log(toBeLogged); +} diff --git a/src/revisions.ts b/src/revisions.ts new file mode 100644 index 0000000000000..9b307e9dca682 --- /dev/null +++ b/src/revisions.ts @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type Revisions = Readonly<{ + readonly chromium: string; + readonly firefox: string; +}>; + +export const PUPPETEER_REVISIONS: Revisions = { + chromium: '756035', + firefox: 'latest', +}; diff --git a/utils/check_availability.js b/utils/check_availability.js index 8cafc14025f3a..1a5f2cab49d84 100755 --- a/utils/check_availability.js +++ b/utils/check_availability.js @@ -17,8 +17,6 @@ const assert = require('assert'); const https = require('https'); - -const packageJSON = require('../package.json'); const BrowserFetcher = require('../lib/BrowserFetcher').BrowserFetcher; const SUPPORTER_PLATFORMS = ['linux', 'mac', 'win32', 'win64']; @@ -163,7 +161,10 @@ async function checkRollCandidate() { stableLinuxInfo.versions[0].branch_base_position, 10 ); - const currentRevision = parseInt(packageJSON.puppeteer.chromium_revision, 10); + const currentRevision = parseInt( + require('../lib/cjs/revisions').PUPPETEER_REVISIONS.chromium, + 10 + ); checkRangeAvailability({ fromRevision: stableLinuxRevision, diff --git a/utils/prepare_puppeteer_core.js b/utils/prepare_puppeteer_core.js index b534dbfd13102..e1e9a64f2f4ed 100755 --- a/utils/prepare_puppeteer_core.js +++ b/utils/prepare_puppeteer_core.js @@ -23,4 +23,5 @@ const json = require(packagePath); json.name = 'puppeteer-core'; delete json.scripts.install; +json.main = './cjs-entry-core.js'; fs.writeFileSync(packagePath, JSON.stringify(json, null, ' '));