From 78c313b4c8f4697c4b198b8d435d66b754a32f44 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Thu, 25 Jun 2020 16:26:51 +0100 Subject: [PATCH 1/5] chore: Don't store revisions in `package.json` It's quite messy to have to require the `package.json` file in multiple places purely to find out what revision of a given browser we want to use. We can also achieve better type safety by placing it in an actual source file. This commit makes that change and also tidies up our reliance on `package.json` within the source code generally; we now only use it to find the location of the Puppeteer root such that we know where to install downloaded browsers to. To avoid using `package.json` to parse the name of the module, we also now explicitly have an entry point for the Puppeteer module and the Puppeter Core module. This will make it easier in the future to ship less code as part of core (e.g. core never needs to download a browser, so why ship that code?). Core can also then not have any revisions based info contained in it. The test install script has also been updated to ensure that puppeteer-core can be installed correctly too. Finally, the `install` script has been moved to TypeScript for nicer typechecking and safety. The functionality of it has not changed. --- README.md | 4 +- cjs-entry-core.js | 29 +++ install.js | 255 ++++--------------- new-docs/puppeteer.puppeteer._projectroot.md | 11 - new-docs/puppeteer.puppeteer.md | 1 - package.json | 10 +- scripts/test-install.sh | 15 ++ src/common/Puppeteer.ts | 10 +- src/index-core.ts | 20 ++ src/index.ts | 17 +- src/initialize.ts | 25 +- src/install.ts | 176 +++++++++++++ src/revisions.ts | 25 ++ utils/check_availability.js | 7 +- utils/prepare_puppeteer_core.js | 1 + 15 files changed, 337 insertions(+), 269 deletions(-) create mode 100644 cjs-entry-core.js delete mode 100644 new-docs/puppeteer.puppeteer._projectroot.md create mode 100644 src/index-core.ts create mode 100644 src/install.ts create mode 100644 src/revisions.ts diff --git a/README.md b/README.md index 7bd432809f21d..e36a82a5b6deb 100644 --- a/README.md +++ b/README.md @@ -380,12 +380,12 @@ npm install puppeteer-core@chrome-71 #### Q: Which Chromium version does Puppeteer use? -Look for `chromium_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section. +Look for the `chromium` entry in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section. #### Q: Which Firefox version does Puppeteer use? -Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json) is `latest` -- Puppeteer isn't tied to a particular Firefox version. +Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox` in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts) is `latest` -- Puppeteer isn't tied to a particular Firefox version. To fetch Firefox Nightly as part of Puppeteer installation: diff --git a/cjs-entry-core.js b/cjs-entry-core.js new file mode 100644 index 0000000000000..efcdb39027f7e --- /dev/null +++ b/cjs-entry-core.js @@ -0,0 +1,29 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * We use `export default puppeteer` in `src/index.ts` to expose the library But + * TypeScript in CJS mode compiles that to `exports.default = `. This means that + * our CJS Node users would have to use `require('puppeteer').default` which + * isn't very nice. + * + * So instead we expose this file as our entry point. This requires the compiled + * Puppeteer output and re-exports the `default` export via `module.exports.` + * This means that we can publish to CJS and ESM whilst maintaining the expected + * import behaviour for CJS and ESM users. + */ +const puppeteerExport = require('./lib/cjs/index-core'); +module.exports = puppeteerExport.default; diff --git a/install.js b/install.js index cf56da410ea48..402a72e0d2f60 100644 --- a/install.js +++ b/install.js @@ -25,221 +25,62 @@ */ const compileTypeScriptIfRequired = require('./typescript-if-required'); -const os = require('os'); - -const firefoxVersions = - 'https://product-details.mozilla.org/1.0/firefox_versions.json'; -const supportedProducts = { - chrome: 'Chromium', - firefox: 'Firefox Nightly', -}; async function download() { await compileTypeScriptIfRequired(); - - const downloadHost = - process.env.PUPPETEER_DOWNLOAD_HOST || - process.env.npm_config_puppeteer_download_host || - process.env.npm_package_config_puppeteer_download_host; - const puppeteer = require('.'); - const product = - process.env.PUPPETEER_PRODUCT || - process.env.npm_config_puppeteer_product || - process.env.npm_package_config_puppeteer_product || - 'chrome'; - const browserFetcher = puppeteer.createBrowserFetcher({ - product, - host: downloadHost, - }); - const revision = await getRevision(); - await fetchBinary(revision); - - function getRevision() { - if (product === 'chrome') { - return ( - process.env.PUPPETEER_CHROMIUM_REVISION || - process.env.npm_config_puppeteer_chromium_revision || - process.env.npm_package_config_puppeteer_chromium_revision || - require('./package.json').puppeteer.chromium_revision - ); - } else if (product === 'firefox') { - puppeteer._preferredRevision = require('./package.json').puppeteer.firefox_revision; - return getFirefoxNightlyVersion(browserFetcher.host()).catch((error) => { - console.error(error); - process.exit(1); - }); - } else { - throw new Error(`Unsupported product ${product}`); - } + // need to ensure TS is compiled before loading the installer + const { downloadBrowser, logPolitely } = require('./lib/cjs/install'); + + if (process.env.PUPPETEER_SKIP_DOWNLOAD) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.' + ); + return; } - - function fetchBinary(revision) { - const revisionInfo = browserFetcher.revisionInfo(revision); - - // Do nothing if the revision is already downloaded. - if (revisionInfo.local) { - logPolitely( - `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` - ); - return; - } - - // Override current environment proxy settings with npm configuration, if any. - const NPM_HTTPS_PROXY = - process.env.npm_config_https_proxy || process.env.npm_config_proxy; - const NPM_HTTP_PROXY = - process.env.npm_config_http_proxy || process.env.npm_config_proxy; - const NPM_NO_PROXY = process.env.npm_config_no_proxy; - - if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; - if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY; - if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY; - - /** - * @param {!Array} - * @returns {!Promise} - */ - function onSuccess(localRevisions) { - if (os.arch() !== 'arm64') { - logPolitely( - `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` - ); - } - localRevisions = localRevisions.filter( - (revision) => revision !== revisionInfo.revision - ); - const cleanupOldVersions = localRevisions.map((revision) => - browserFetcher.remove(revision) - ); - Promise.all([...cleanupOldVersions]); - } - - /** - * @param {!Error} error - */ - function onError(error) { - console.error( - `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` - ); - console.error(error); - process.exit(1); - } - - let progressBar = null; - let lastDownloadedBytes = 0; - function onProgress(downloadedBytes, totalBytes) { - if (!progressBar) { - const ProgressBar = require('progress'); - progressBar = new ProgressBar( - `Downloading ${ - supportedProducts[product] - } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - } - - return browserFetcher - .download(revisionInfo.revision, onProgress) - .then(() => browserFetcher.localRevisions()) - .then(onSuccess) - .catch(onError); + if ( + process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD || + process.env.npm_config_puppeteer_skip_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.' + ); + return; } - - function toMegabytes(bytes) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; + if ( + process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD || + process.env.npm_package_config_puppeteer_skip_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.' + ); + return; } - - function getFirefoxNightlyVersion(host) { - const https = require('https'); - const promise = new Promise((resolve, reject) => { - let data = ''; - logPolitely(`Requesting latest Firefox Nightly version from ${host}`); - https - .get(firefoxVersions, (r) => { - if (r.statusCode >= 400) - return reject(new Error(`Got status code ${r.statusCode}`)); - r.on('data', (chunk) => { - data += chunk; - }); - r.on('end', () => { - try { - const versions = JSON.parse(data); - return resolve(versions.FIREFOX_NIGHTLY); - } catch { - return reject(new Error('Firefox version not found')); - } - }); - }) - .on('error', reject); - }); - return promise; + if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.' + ); + return; + } + if ( + process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || + process.env.npm_config_puppeteer_skip_chromium_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.' + ); + return; + } + if ( + process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || + process.env.npm_package_config_puppeteer_skip_chromium_download + ) { + logPolitely( + '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.' + ); + return; } -} - -function logPolitely(toBeLogged) { - const logLevel = process.env.npm_config_loglevel; - const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; - - if (!logLevelDisplay) console.log(toBeLogged); -} -if (process.env.PUPPETEER_SKIP_DOWNLOAD) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.' - ); - return; -} -if ( - process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD || - process.env.npm_config_puppeteer_skip_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.' - ); - return; -} -if ( - process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD || - process.env.npm_package_config_puppeteer_skip_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.' - ); - return; -} -if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.' - ); - return; -} -if ( - process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || - process.env.npm_config_puppeteer_skip_chromium_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.' - ); - return; -} -if ( - process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD || - process.env.npm_package_config_puppeteer_skip_chromium_download -) { - logPolitely( - '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.' - ); - return; + downloadBrowser(); } download(); diff --git a/new-docs/puppeteer.puppeteer._projectroot.md b/new-docs/puppeteer.puppeteer._projectroot.md deleted file mode 100644 index 215226cc8552f..0000000000000 --- a/new-docs/puppeteer.puppeteer._projectroot.md +++ /dev/null @@ -1,11 +0,0 @@ - - -[Home](./index.md) > [puppeteer](./puppeteer.md) > [Puppeteer](./puppeteer.puppeteer.md) > [\_projectRoot](./puppeteer.puppeteer._projectroot.md) - -## Puppeteer.\_projectRoot property - -Signature: - -```typescript -_projectRoot: string; -``` diff --git a/new-docs/puppeteer.puppeteer.md b/new-docs/puppeteer.puppeteer.md index f27df977800aa..e96e126c0bb3d 100644 --- a/new-docs/puppeteer.puppeteer.md +++ b/new-docs/puppeteer.puppeteer.md @@ -42,7 +42,6 @@ const puppeteer = require('puppeteer'); | [\_isPuppeteerCore](./puppeteer.puppeteer._ispuppeteercore.md) | | boolean | | | [\_lazyLauncher](./puppeteer.puppeteer._lazylauncher.md) | | ProductLauncher | | | [\_preferredRevision](./puppeteer.puppeteer._preferredrevision.md) | | string | | -| [\_projectRoot](./puppeteer.puppeteer._projectroot.md) | | string | | | [devices](./puppeteer.puppeteer.devices.md) | | DevicesMap | | | [errors](./puppeteer.puppeteer.errors.md) | | [PuppeteerErrors](./puppeteer.puppeteererrors.md) | | | [product](./puppeteer.puppeteer.product.md) | | string | | diff --git a/package.json b/package.json index 565425af2ee39..be0a4dda42462 100644 --- a/package.json +++ b/package.json @@ -7,10 +7,6 @@ "engines": { "node": ">=10.18.1" }, - "puppeteer": { - "chromium_revision": "756035", - "firefox_revision": "latest" - }, "scripts": { "unit": "tsc --version && mocha --config mocha-config/puppeteer-unit-tests.js", "unit-with-coverage": "cross-env COVERAGE=1 npm run unit", @@ -41,10 +37,10 @@ }, "files": [ "lib/", - "index.js", "install.js", "typescript-if-required.js", - "cjs-entry.js" + "cjs-entry.js", + "cjs-entry-core.js" ], "author": "The Chromium Authors", "license": "Apache-2.0", @@ -54,9 +50,9 @@ "https-proxy-agent": "^4.0.0", "mime": "^2.0.3", "mitt": "^2.0.1", + "pkg-dir": "^4.2.0", "progress": "^2.0.1", "proxy-from-env": "^1.0.0", - "read-pkg-up": "^7.0.1", "rimraf": "^3.0.2", "tar-fs": "^2.0.0", "unbzip2-stream": "^1.3.3", diff --git a/scripts/test-install.sh b/scripts/test-install.sh index 2782d2acff224..f77eb9e021397 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -22,3 +22,18 @@ PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" rm "${tarball}" ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-79.0a1/firefox/firefox + +# Again for puppeteer-core +node ./utils/prepare_puppeteer_core.js +npm pack +tarball="$(realpath puppeteer-core-*.tgz)" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +# Check we can install from the tarball. +# This emulates installing from npm and ensures that: +# 1. we publish the right files in the `files` list from package.json +# 2. The install script works and correctly exits without errors +# 3. Requiring Puppeteer Core from Node works. +npm install --loglevel silent "${tarball}" +node --eval="require('puppeteer-core')" + diff --git a/src/common/Puppeteer.ts b/src/common/Puppeteer.ts index 9cb299a20d76c..b43beb973975d 100644 --- a/src/common/Puppeteer.ts +++ b/src/common/Puppeteer.ts @@ -23,8 +23,6 @@ import { ProductLauncher } from '../node/Launcher'; import { BrowserFetcher, BrowserFetcherOptions } from '../node/BrowserFetcher'; import { puppeteerErrors, PuppeteerErrors } from './Errors'; import { ConnectionTransport } from './ConnectionTransport'; -import readPkgUp from 'read-pkg-up'; - import { devicesMap } from './DeviceDescriptors'; import { DevicesMap } from './DeviceDescriptors'; import { Browser } from './Browser'; @@ -35,6 +33,7 @@ import { clearQueryHandlers, QueryHandler, } from './QueryHandler'; +import { PUPPETEER_REVISIONS } from '../revisions'; /** * The main Puppeteer class @@ -58,7 +57,7 @@ import { * @public */ export class Puppeteer { - _projectRoot: string; + private _projectRoot: string; _preferredRevision: string; _isPuppeteerCore: boolean; _changedProduct = false; @@ -173,14 +172,13 @@ export class Puppeteer { this._lazyLauncher.product !== this._productName || this._changedProduct ) { - const { packageJson } = readPkgUp.sync({ cwd: __dirname }); switch (this._productName) { case 'firefox': - this._preferredRevision = packageJson.puppeteer.firefox_revision; + this._preferredRevision = PUPPETEER_REVISIONS.firefox; break; case 'chrome': default: - this._preferredRevision = packageJson.puppeteer.chromium_revision; + this._preferredRevision = PUPPETEER_REVISIONS.chromium; } this._changedProduct = false; this._lazyLauncher = Launcher( diff --git a/src/index-core.ts b/src/index-core.ts new file mode 100644 index 0000000000000..692279f1273f9 --- /dev/null +++ b/src/index-core.ts @@ -0,0 +1,20 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { initializePuppeteer } from './initialize'; + +const puppeteer = initializePuppeteer('puppeteer-core'); +export default puppeteer; diff --git a/src/index.ts b/src/index.ts index c395a7690ae14..bcabc49357841 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,20 +14,7 @@ * limitations under the License. */ -import { initializePuppeteer, InitOptions } from './initialize'; -import * as path from 'path'; -import readPkgUp from 'read-pkg-up'; - -const packageJsonResult = readPkgUp.sync({ - cwd: __dirname, -}); -const packageJson = packageJsonResult.packageJson as unknown; - -const rootDir = path.dirname(packageJsonResult.path); - -const puppeteer = initializePuppeteer({ - packageJson: packageJson as InitOptions['packageJson'], - rootDirectory: rootDir, -}); +import { initializePuppeteer } from './initialize'; +const puppeteer = initializePuppeteer('puppeteer'); export default puppeteer; diff --git a/src/initialize.ts b/src/initialize.ts index 4cf16ecf4b798..429379c844c4c 100644 --- a/src/initialize.ts +++ b/src/initialize.ts @@ -21,28 +21,19 @@ const api = require('./api'); import { helper } from './common/helper'; import { Puppeteer } from './common/Puppeteer'; +import { PUPPETEER_REVISIONS } from './revisions'; +import pkgDir from 'pkg-dir'; -export interface InitOptions { - packageJson: { - puppeteer: { - chromium_revision: string; - firefox_revision: string; - }; - name: string; - }; - rootDirectory: string; -} - -export const initializePuppeteer = (options: InitOptions): Puppeteer => { - const { packageJson, rootDirectory } = options; +export const initializePuppeteer = (packageName: string): Puppeteer => { + const puppeteerRootDirectory = pkgDir.sync(__dirname); for (const className in api) { if (typeof api[className] === 'function') helper.installAsyncStackHooks(api[className]); } - let preferredRevision = packageJson.puppeteer.chromium_revision; - const isPuppeteerCore = packageJson.name === 'puppeteer-core'; + let preferredRevision = PUPPETEER_REVISIONS.chromium; + const isPuppeteerCore = packageName === 'puppeteer-core'; // puppeteer-core ignores environment variables const product = isPuppeteerCore ? undefined @@ -50,10 +41,10 @@ export const initializePuppeteer = (options: InitOptions): Puppeteer => { process.env.npm_config_puppeteer_product || process.env.npm_package_config_puppeteer_product; if (!isPuppeteerCore && product === 'firefox') - preferredRevision = packageJson.puppeteer.firefox_revision; + preferredRevision = PUPPETEER_REVISIONS.firefox; const puppeteer = new Puppeteer( - rootDirectory, + puppeteerRootDirectory, preferredRevision, isPuppeteerCore, product diff --git a/src/install.ts b/src/install.ts new file mode 100644 index 0000000000000..57872e93ccaa9 --- /dev/null +++ b/src/install.ts @@ -0,0 +1,176 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import https from 'https'; +import ProgressBar from 'progress'; +import puppeteer from './index'; +import { PUPPETEER_REVISIONS } from './revisions'; + +const firefoxVersions = + 'https://product-details.mozilla.org/1.0/firefox_versions.json'; + +const supportedProducts = { + chrome: 'Chromium', + firefox: 'Firefox Nightly', +} as const; + +export async function downloadBrowser() { + const downloadHost = + process.env.PUPPETEER_DOWNLOAD_HOST || + process.env.npm_config_puppeteer_download_host || + process.env.npm_package_config_puppeteer_download_host; + const product = + process.env.PUPPETEER_PRODUCT || + process.env.npm_config_puppeteer_product || + process.env.npm_package_config_puppeteer_product || + 'chrome'; + const browserFetcher = puppeteer.createBrowserFetcher({ + product, + host: downloadHost, + }); + const revision = await getRevision(); + await fetchBinary(revision); + + function getRevision() { + if (product === 'chrome') { + return ( + process.env.PUPPETEER_CHROMIUM_REVISION || + process.env.npm_config_puppeteer_chromium_revision || + PUPPETEER_REVISIONS.chromium + ); + } else if (product === 'firefox') { + puppeteer._preferredRevision = PUPPETEER_REVISIONS.firefox; + return getFirefoxNightlyVersion(browserFetcher.host()).catch((error) => { + console.error(error); + process.exit(1); + }); + } else { + throw new Error(`Unsupported product ${product}`); + } + } + + function fetchBinary(revision) { + const revisionInfo = browserFetcher.revisionInfo(revision); + + // Do nothing if the revision is already downloaded. + if (revisionInfo.local) { + logPolitely( + `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.` + ); + return; + } + + // Override current environment proxy settings with npm configuration, if any. + const NPM_HTTPS_PROXY = + process.env.npm_config_https_proxy || process.env.npm_config_proxy; + const NPM_HTTP_PROXY = + process.env.npm_config_http_proxy || process.env.npm_config_proxy; + const NPM_NO_PROXY = process.env.npm_config_no_proxy; + + if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY; + if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY; + if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY; + + function onSuccess(localRevisions: string[]): void { + if (os.arch() !== 'arm64') { + logPolitely( + `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}` + ); + } + localRevisions = localRevisions.filter( + (revision) => revision !== revisionInfo.revision + ); + const cleanupOldVersions = localRevisions.map((revision) => + browserFetcher.remove(revision) + ); + Promise.all([...cleanupOldVersions]); + } + + function onError(error: Error) { + console.error( + `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.` + ); + console.error(error); + process.exit(1); + } + + let progressBar = null; + let lastDownloadedBytes = 0; + function onProgress(downloadedBytes, totalBytes) { + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${ + supportedProducts[product] + } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + } + + return browserFetcher + .download(revisionInfo.revision, onProgress) + .then(() => browserFetcher.localRevisions()) + .then(onSuccess) + .catch(onError); + } + + function toMegabytes(bytes) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; + } + + function getFirefoxNightlyVersion(host) { + const promise = new Promise((resolve, reject) => { + let data = ''; + logPolitely(`Requesting latest Firefox Nightly version from ${host}`); + https + .get(firefoxVersions, (r) => { + if (r.statusCode >= 400) + return reject(new Error(`Got status code ${r.statusCode}`)); + r.on('data', (chunk) => { + data += chunk; + }); + r.on('end', () => { + try { + const versions = JSON.parse(data); + return resolve(versions.FIREFOX_NIGHTLY); + } catch { + return reject(new Error('Firefox version not found')); + } + }); + }) + .on('error', reject); + }); + return promise; + } +} + +export function logPolitely(toBeLogged) { + const logLevel = process.env.npm_config_loglevel; + const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1; + + // eslint-disable-next-line no-console + if (!logLevelDisplay) console.log(toBeLogged); +} diff --git a/src/revisions.ts b/src/revisions.ts new file mode 100644 index 0000000000000..9b307e9dca682 --- /dev/null +++ b/src/revisions.ts @@ -0,0 +1,25 @@ +/** + * Copyright 2020 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type Revisions = Readonly<{ + readonly chromium: string; + readonly firefox: string; +}>; + +export const PUPPETEER_REVISIONS: Revisions = { + chromium: '756035', + firefox: 'latest', +}; diff --git a/utils/check_availability.js b/utils/check_availability.js index 8cafc14025f3a..1a5f2cab49d84 100755 --- a/utils/check_availability.js +++ b/utils/check_availability.js @@ -17,8 +17,6 @@ const assert = require('assert'); const https = require('https'); - -const packageJSON = require('../package.json'); const BrowserFetcher = require('../lib/BrowserFetcher').BrowserFetcher; const SUPPORTER_PLATFORMS = ['linux', 'mac', 'win32', 'win64']; @@ -163,7 +161,10 @@ async function checkRollCandidate() { stableLinuxInfo.versions[0].branch_base_position, 10 ); - const currentRevision = parseInt(packageJSON.puppeteer.chromium_revision, 10); + const currentRevision = parseInt( + require('../lib/cjs/revisions').PUPPETEER_REVISIONS.chromium, + 10 + ); checkRangeAvailability({ fromRevision: stableLinuxRevision, diff --git a/utils/prepare_puppeteer_core.js b/utils/prepare_puppeteer_core.js index b534dbfd13102..e1e9a64f2f4ed 100755 --- a/utils/prepare_puppeteer_core.js +++ b/utils/prepare_puppeteer_core.js @@ -23,4 +23,5 @@ const json = require(packagePath); json.name = 'puppeteer-core'; delete json.scripts.install; +json.main = './cjs-entry-core.js'; fs.writeFileSync(packagePath, JSON.stringify(json, null, ' ')); From f45d94c7f329c631ded218165532953f8fe21c54 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Mon, 29 Jun 2020 15:20:18 +0100 Subject: [PATCH 2/5] Fix location of puppeteer core build --- scripts/test-install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test-install.sh b/scripts/test-install.sh index f77eb9e021397..be41733a6b280 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -1,6 +1,7 @@ #!/usr/bin/env sh set -e +ROOTDIR="$(pwd)" # Pack the module into a tarball npm pack tarball="$(realpath puppeteer-*.tgz)" @@ -24,8 +25,10 @@ rm "${tarball}" ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-79.0a1/firefox/firefox # Again for puppeteer-core +cd $ROOTDIR node ./utils/prepare_puppeteer_core.js npm pack +git reset --hard tarball="$(realpath puppeteer-core-*.tgz)" TMPDIR="$(mktemp -d)" cd $TMPDIR From e7f1a69d3d453707f38e40cfce7460037261c068 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Mon, 29 Jun 2020 15:20:47 +0100 Subject: [PATCH 3/5] Ignore any local tarballs from test script --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 89804cb891757..c5f0b19c647cc 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ yarn.lock test/coverage.json temp/ dependency-chart.png +puppeteer-core-*.tgz From 5c84bcf150200a67bfe3926bfe11f2ed28169cf3 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Mon, 29 Jun 2020 15:28:53 +0100 Subject: [PATCH 4/5] Remove bad git reset --- scripts/test-install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test-install.sh b/scripts/test-install.sh index be41733a6b280..76703abcb492f 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -28,7 +28,6 @@ ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-79.0a1/firefox/firefox cd $ROOTDIR node ./utils/prepare_puppeteer_core.js npm pack -git reset --hard tarball="$(realpath puppeteer-core-*.tgz)" TMPDIR="$(mktemp -d)" cd $TMPDIR From 59c398c91838d88b220a960255011a3480b596f0 Mon Sep 17 00:00:00 2001 From: Jack Franklin Date: Mon, 29 Jun 2020 15:45:49 +0100 Subject: [PATCH 5/5] travis change --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 467f88577b8c1..8da18b9a8579a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,11 +64,19 @@ jobs: - CHROMIUM=true script: - npm run compare-protocol-d-ts - - npm run test-install - npm run lint - npm run test-doclint - npm run ensure-new-docs-up-to-date + # This bot runs separately as it changes package.json to test puppeteer-core + # and we don't want that leaking into other bots and causing issues. + - node_js: "10.19.0" + name: 'Test bundling and install of packages' + env: + - CHROMIUM=true + script: + - npm run test-install + # Runs unit tests on Linux + Firefox - node_js: "10.19.0" name: 'Unit tests: Linux/Firefox'