chore: Don't store revisions in package.json (#6109)

* chore: Don't store revisions in `package.json` It's quite messy to have to require the `package.json` file in multiple places purely to find out what revision of a given browser we want to use. We can also achieve better type safety by placing it in an actual source file. This commit makes that change and also tidies up our reliance on `package.json` within the source code generally; we now only use it to find the location of the Puppeteer root such that we know where to install downloaded browsers to. To avoid using `package.json` to parse the name of the module, we also now explicitly have an entry point for the Puppeteer module and the Puppeter Core module. This will make it easier in the future to ship less code as part of core (e.g. core never needs to download a browser, so why ship that code?). Core can also then not have any revisions based info contained in it. The test install script has also been updated to ensure that puppeteer-core can be installed correctly too. Finally, the `install` script has been moved to TypeScript for nicer typechecking and safety. The functionality of it has not changed.
puppeteer · Jun 29, 2020 · 1f5e333 · 1f5e333
1 parent 123c377
commit 1f5e333
Show file tree

Hide file tree

Showing 17 changed files with 349 additions and 270 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,3 +17,4 @@ yarn.lock
 test/coverage.json
 temp/
 dependency-chart.png
+puppeteer-core-*.tgz
diff --git a/.travis.yml b/.travis.yml
@@ -64,11 +64,19 @@ jobs:
         - CHROMIUM=true
       script:
         - npm run compare-protocol-d-ts
-        - npm run test-install
         - npm run lint
         - npm run test-doclint
         - npm run ensure-new-docs-up-to-date
 
+    # This bot runs separately as it changes package.json to test puppeteer-core
+    # and we don't want that leaking into other bots and causing issues.
+    - node_js: "10.19.0"
+      name: 'Test bundling and install of packages'
+      env:
+        - CHROMIUM=true
+      script:
+        - npm run test-install
+
     # Runs unit tests on Linux + Firefox
     - node_js: "10.19.0"
       name: 'Unit tests: Linux/Firefox'

diff --git a/README.md b/README.md
@@ -380,12 +380,12 @@ npm install puppeteer-core@chrome-71
 
 #### Q: Which Chromium version does Puppeteer use?
 
-Look for `chromium_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section.
+Look for the `chromium` entry in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts). To find the corresponding Chromium commit and version number, search for the revision prefixed by an `r` in [OmahaProxy](https://omahaproxy.appspot.com/)'s "Find Releases" section.
 
 
 #### Q: Which Firefox version does Puppeteer use?
 
-Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox_revision` in [package.json](https://github.com/puppeteer/puppeteer/blob/main/package.json) is `latest` -- Puppeteer isn't tied to a particular Firefox version.
+Since Firefox support is experimental, Puppeteer downloads the latest [Firefox Nightly](https://wiki.mozilla.org/Nightly) when the `PUPPETEER_PRODUCT` environment variable is set to `firefox`. That's also why the value of `firefox` in [revisions.ts](https://github.com/puppeteer/puppeteer/blob/main/src/revisions.ts) is `latest` -- Puppeteer isn't tied to a particular Firefox version.
 
 To fetch Firefox Nightly as part of Puppeteer installation:
 

diff --git a/cjs-entry-core.js b/cjs-entry-core.js
@@ -0,0 +1,29 @@
+/**
+ * Copyright 2020 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * We use `export default puppeteer` in `src/index.ts` to expose the library But
+ * TypeScript in CJS mode compiles that to `exports.default = `. This means that
+ * our CJS Node users would have to use `require('puppeteer').default` which
+ * isn't very nice.
+ *
+ * So instead we expose this file as our entry point. This requires the compiled
+ * Puppeteer output and re-exports the `default` export via `module.exports.`
+ * This means that we can publish to CJS and ESM whilst maintaining the expected
+ * import behaviour for CJS and ESM users.
+ */
+const puppeteerExport = require('./lib/cjs/index-core');
+module.exports = puppeteerExport.default;
diff --git a/install.js b/install.js
@@ -25,221 +25,62 @@
  */
 
 const compileTypeScriptIfRequired = require('./typescript-if-required');
-const os = require('os');
-
-const firefoxVersions =
-  'https://product-details.mozilla.org/1.0/firefox_versions.json';
-const supportedProducts = {
-  chrome: 'Chromium',
-  firefox: 'Firefox Nightly',
-};
 
 async function download() {
   await compileTypeScriptIfRequired();
-
-  const downloadHost =
-    process.env.PUPPETEER_DOWNLOAD_HOST ||
-    process.env.npm_config_puppeteer_download_host ||
-    process.env.npm_package_config_puppeteer_download_host;
-  const puppeteer = require('.');
-  const product =
-    process.env.PUPPETEER_PRODUCT ||
-    process.env.npm_config_puppeteer_product ||
-    process.env.npm_package_config_puppeteer_product ||
-    'chrome';
-  const browserFetcher = puppeteer.createBrowserFetcher({
-    product,
-    host: downloadHost,
-  });
-  const revision = await getRevision();
-  await fetchBinary(revision);
-
-  function getRevision() {
-    if (product === 'chrome') {
-      return (
-        process.env.PUPPETEER_CHROMIUM_REVISION ||
-        process.env.npm_config_puppeteer_chromium_revision ||
-        process.env.npm_package_config_puppeteer_chromium_revision ||
-        require('./package.json').puppeteer.chromium_revision
-      );
-    } else if (product === 'firefox') {
-      puppeteer._preferredRevision = require('./package.json').puppeteer.firefox_revision;
-      return getFirefoxNightlyVersion(browserFetcher.host()).catch((error) => {
-        console.error(error);
-        process.exit(1);
-      });
-    } else {
-      throw new Error(`Unsupported product ${product}`);
-    }
+  // need to ensure TS is compiled before loading the installer
+  const { downloadBrowser, logPolitely } = require('./lib/cjs/install');
+
+  if (process.env.PUPPETEER_SKIP_DOWNLOAD) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.'
+    );
+    return;
   }
-
-  function fetchBinary(revision) {
-    const revisionInfo = browserFetcher.revisionInfo(revision);
-
-    // Do nothing if the revision is already downloaded.
-    if (revisionInfo.local) {
-      logPolitely(
-        `${supportedProducts[product]} is already in ${revisionInfo.folderPath}; skipping download.`
-      );
-      return;
-    }
-
-    // Override current environment proxy settings with npm configuration, if any.
-    const NPM_HTTPS_PROXY =
-      process.env.npm_config_https_proxy || process.env.npm_config_proxy;
-    const NPM_HTTP_PROXY =
-      process.env.npm_config_http_proxy || process.env.npm_config_proxy;
-    const NPM_NO_PROXY = process.env.npm_config_no_proxy;
-
-    if (NPM_HTTPS_PROXY) process.env.HTTPS_PROXY = NPM_HTTPS_PROXY;
-    if (NPM_HTTP_PROXY) process.env.HTTP_PROXY = NPM_HTTP_PROXY;
-    if (NPM_NO_PROXY) process.env.NO_PROXY = NPM_NO_PROXY;
-
-    /**
-     * @param {!Array<string>}
-     * @returns {!Promise}
-     */
-    function onSuccess(localRevisions) {
-      if (os.arch() !== 'arm64') {
-        logPolitely(
-          `${supportedProducts[product]} (${revisionInfo.revision}) downloaded to ${revisionInfo.folderPath}`
-        );
-      }
-      localRevisions = localRevisions.filter(
-        (revision) => revision !== revisionInfo.revision
-      );
-      const cleanupOldVersions = localRevisions.map((revision) =>
-        browserFetcher.remove(revision)
-      );
-      Promise.all([...cleanupOldVersions]);
-    }
-
-    /**
-     * @param {!Error} error
-     */
-    function onError(error) {
-      console.error(
-        `ERROR: Failed to set up ${supportedProducts[product]} r${revision}! Set "PUPPETEER_SKIP_DOWNLOAD" env variable to skip download.`
-      );
-      console.error(error);
-      process.exit(1);
-    }
-
-    let progressBar = null;
-    let lastDownloadedBytes = 0;
-    function onProgress(downloadedBytes, totalBytes) {
-      if (!progressBar) {
-        const ProgressBar = require('progress');
-        progressBar = new ProgressBar(
-          `Downloading ${
-            supportedProducts[product]
-          } r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `,
-          {
-            complete: '=',
-            incomplete: ' ',
-            width: 20,
-            total: totalBytes,
-          }
-        );
-      }
-      const delta = downloadedBytes - lastDownloadedBytes;
-      lastDownloadedBytes = downloadedBytes;
-      progressBar.tick(delta);
-    }
-
-    return browserFetcher
-      .download(revisionInfo.revision, onProgress)
-      .then(() => browserFetcher.localRevisions())
-      .then(onSuccess)
-      .catch(onError);
+  if (
+    process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD ||
+    process.env.npm_config_puppeteer_skip_download
+  ) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.'
+    );
+    return;
   }
-
-  function toMegabytes(bytes) {
-    const mb = bytes / 1024 / 1024;
-    return `${Math.round(mb * 10) / 10} Mb`;
+  if (
+    process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD ||
+    process.env.npm_package_config_puppeteer_skip_download
+  ) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.'
+    );
+    return;
   }
-
-  function getFirefoxNightlyVersion(host) {
-    const https = require('https');
-    const promise = new Promise((resolve, reject) => {
-      let data = '';
-      logPolitely(`Requesting latest Firefox Nightly version from ${host}`);
-      https
-        .get(firefoxVersions, (r) => {
-          if (r.statusCode >= 400)
-            return reject(new Error(`Got status code ${r.statusCode}`));
-          r.on('data', (chunk) => {
-            data += chunk;
-          });
-          r.on('end', () => {
-            try {
-              const versions = JSON.parse(data);
-              return resolve(versions.FIREFOX_NIGHTLY);
-            } catch {
-              return reject(new Error('Firefox version not found'));
-            }
-          });
-        })
-        .on('error', reject);
-    });
-    return promise;
+  if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.'
+    );
+    return;
+  }
+  if (
+    process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD ||
+    process.env.npm_config_puppeteer_skip_chromium_download
+  ) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.'
+    );
+    return;
+  }
+  if (
+    process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD ||
+    process.env.npm_package_config_puppeteer_skip_chromium_download
+  ) {
+    logPolitely(
+      '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.'
+    );
+    return;
   }
-}
-
-function logPolitely(toBeLogged) {
-  const logLevel = process.env.npm_config_loglevel;
-  const logLevelDisplay = ['silent', 'error', 'warn'].indexOf(logLevel) > -1;
-
-  if (!logLevelDisplay) console.log(toBeLogged);
-}
 
-if (process.env.PUPPETEER_SKIP_DOWNLOAD) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" environment variable was found.'
-  );
-  return;
-}
-if (
-  process.env.NPM_CONFIG_PUPPETEER_SKIP_DOWNLOAD ||
-  process.env.npm_config_puppeteer_skip_download
-) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in npm config.'
-  );
-  return;
-}
-if (
-  process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_DOWNLOAD ||
-  process.env.npm_package_config_puppeteer_skip_download
-) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_DOWNLOAD" was set in project config.'
-  );
-  return;
-}
-if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" environment variable was found.'
-  );
-  return;
-}
-if (
-  process.env.NPM_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD ||
-  process.env.npm_config_puppeteer_skip_chromium_download
-) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in npm config.'
-  );
-  return;
-}
-if (
-  process.env.NPM_PACKAGE_CONFIG_PUPPETEER_SKIP_CHROMIUM_DOWNLOAD ||
-  process.env.npm_package_config_puppeteer_skip_chromium_download
-) {
-  logPolitely(
-    '**INFO** Skipping browser download. "PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" was set in project config.'
-  );
-  return;
+  downloadBrowser();
 }
 
 download();
diff --git a/new-docs/puppeteer.puppeteer._projectroot.md b/new-docs/puppeteer.puppeteer._projectroot.md
diff --git a/new-docs/puppeteer.puppeteer.md b/new-docs/puppeteer.puppeteer.md
@@ -42,7 +42,6 @@ const puppeteer = require('puppeteer');
 |  [\_isPuppeteerCore](./puppeteer.puppeteer._ispuppeteercore.md) |  | boolean |  |
 |  [\_lazyLauncher](./puppeteer.puppeteer._lazylauncher.md) |  | ProductLauncher |  |
 |  [\_preferredRevision](./puppeteer.puppeteer._preferredrevision.md) |  | string |  |
-|  [\_projectRoot](./puppeteer.puppeteer._projectroot.md) |  | string |  |
 |  [devices](./puppeteer.puppeteer.devices.md) |  | DevicesMap |  |
 |  [errors](./puppeteer.puppeteer.errors.md) |  | [PuppeteerErrors](./puppeteer.puppeteererrors.md) |  |
 |  [product](./puppeteer.puppeteer.product.md) |  | string |  |

diff --git a/package.json b/package.json
@@ -7,10 +7,6 @@
   "engines": {
     "node": ">=10.18.1"
   },
-  "puppeteer": {
-    "chromium_revision": "756035",
-    "firefox_revision": "latest"
-  },
   "scripts": {
     "unit": "tsc --version && mocha --config mocha-config/puppeteer-unit-tests.js",
     "unit-with-coverage": "cross-env COVERAGE=1 npm run unit",
@@ -41,10 +37,10 @@
   },
   "files": [
     "lib/",
-    "index.js",
     "install.js",
     "typescript-if-required.js",
-    "cjs-entry.js"
+    "cjs-entry.js",
+    "cjs-entry-core.js"
   ],
   "author": "The Chromium Authors",
   "license": "Apache-2.0",
@@ -54,9 +50,9 @@
     "https-proxy-agent": "^4.0.0",
     "mime": "^2.0.3",
     "mitt": "^2.0.1",
+    "pkg-dir": "^4.2.0",
     "progress": "^2.0.1",
     "proxy-from-env": "^1.0.0",
-    "read-pkg-up": "^7.0.1",
     "rimraf": "^3.0.2",
     "tar-fs": "^2.0.0",
     "unbzip2-stream": "^1.3.3",