diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a9e7e70bb6852..0d01cb61e89c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: matrix: # Include all major maintenance + active LTS + current Node.js versions. # https://github.com/nodejs/Release#release-schedule - node: [12, 14, 16] + node: [14, 16] steps: - name: Checkout uses: actions/checkout@v3 @@ -111,7 +111,7 @@ jobs: with: # Test only the oldest maintenance LTS Node.js version. # https://github.com/nodejs/Release#release-schedule - node-version: 12 + node-version: 14 - name: Install dependencies run: | @@ -152,7 +152,7 @@ jobs: with: # Test only the oldest maintenance LTS Node.js version. # https://github.com/nodejs/Release#release-schedule - node-version: 12 + node-version: 14 - name: Install dependencies run: | diff --git a/.gitignore b/.gitignore index e6096db96736f..ae3dfc539b1e6 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ yarn.lock test/coverage.json temp/ new-docs/ -puppeteer.tgz +puppeteer*.tgz docs-api-json/ docs-dist/ website/docs diff --git a/compat/README.md b/compat/README.md new file mode 100644 index 0000000000000..a72ecab4e8a3c --- /dev/null +++ b/compat/README.md @@ -0,0 +1,16 @@ +# Compatibility layer + +This directory provides an additional compatibility layer between ES modules and CommonJS. + +## Why? + +Both `./cjs/compat.ts` and `./esm/compat.ts` are written as ES modules, but `./cjs/compat.ts` can additionally use NodeJS CommonJS globals such as `__dirname` and `require` while these are disabled in ES module mode. For more information, see [Differences between ES modules and CommonJS](https://nodejs.org/api/esm.html#differences-between-es-modules-and-commonjs). + +## Adding exports + +In order to add exports, two things need to be done: + +- The exports must be declared in `src/compat.ts`. +- The exports must be realized in `./cjs/compat.ts` and `./esm/compat.ts`. + +In the event `compat.ts` becomes too large, you can place declarations in another file. Just make sure `./cjs`, `./esm`, and `src` have the same structure. diff --git a/compat/cjs/compat.ts b/compat/cjs/compat.ts new file mode 100644 index 0000000000000..6efa3fe9aabe3 --- /dev/null +++ b/compat/cjs/compat.ts @@ -0,0 +1,16 @@ +import { dirname } from 'path'; + +let puppeteerDirname: string; + +try { + // In some environments, like esbuild, this will throw an error. + // We suppress the error since the bundled binary is not expected + // to be used or installed in this case and, therefore, the + // root directory does not have to be known. + puppeteerDirname = dirname(require.resolve('./compat')); +} catch (error) { + // Fallback to __dirname. + puppeteerDirname = __dirname; +} + +export { puppeteerDirname }; diff --git a/compat/cjs/tsconfig.json b/compat/cjs/tsconfig.json new file mode 100644 index 0000000000000..2bcb2b984b4c8 --- /dev/null +++ b/compat/cjs/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "../../lib/cjs/puppeteer", + "module": "CommonJS" + }, + "references": [{ "path": "../../vendor/tsconfig.cjs.json" }] +} diff --git a/compat/esm/compat.ts b/compat/esm/compat.ts new file mode 100644 index 0000000000000..245c09652e803 --- /dev/null +++ b/compat/esm/compat.ts @@ -0,0 +1,20 @@ +import { createRequire } from 'module'; +import { dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const require = createRequire(import.meta.url); + +let puppeteerDirname: string; + +try { + // In some environments, like esbuild, this will throw an error. + // We suppress the error since the bundled binary is not expected + // to be used or installed in this case and, therefore, the + // root directory does not have to be known. + puppeteerDirname = dirname(require.resolve('./compat')); +} catch (error) { + // Fallback to __dirname. + puppeteerDirname = dirname(fileURLToPath(import.meta.url)); +} + +export { puppeteerDirname }; diff --git a/compat/esm/tsconfig.json b/compat/esm/tsconfig.json new file mode 100644 index 0000000000000..42b320fcd3f28 --- /dev/null +++ b/compat/esm/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "../../lib/esm/puppeteer", + "module": "esnext" + }, + "references": [{ "path": "../../vendor/tsconfig.esm.json" }] +} diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 688a96e65fc13..c139d1a4d94d7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -156,7 +156,7 @@ If you get an error that looks like this when trying to launch Chromium: spawn /Users/.../node_modules/puppeteer/.local-chromium/mac-756035/chrome-mac/Chromium.app/Contents/MacOS/Chromium ENOENT ``` -This means that the browser was downloaded but failed to be extracted correctly. The most common cause is a bug in Node.js v14.0.0 which broke `extract-zip`, the module Puppeteer uses to extract browser downloads into the right place. The bug was fixed in Node.js v14.1.0, so please make sure you're running that version or higher. Alternatively, if you cannot upgrade, you could downgrade to Node.js v12, but we recommend upgrading when possible. +This means that the browser was downloaded but failed to be extracted correctly. The most common cause is a bug in Node.js v14.0.0 which broke `extract-zip`, the module Puppeteer uses to extract browser downloads into the right place. The bug was fixed in Node.js v14.1.0, so please make sure you're running that version or higher. ## Setting Up Chrome Linux Sandbox @@ -242,7 +242,7 @@ Running Puppeteer smoothly on CircleCI requires the following steps: like so: ```yaml docker: - - image: circleci/node:12 # Use your desired version + - image: circleci/node:14 # Use your desired version environment: NODE_ENV: development # Only needed if puppeteer is in `devDependencies` ``` @@ -277,7 +277,7 @@ To fix, you'll need to install the missing dependencies and the latest Chromium package in your Dockerfile: ```Dockerfile -FROM node:12-slim +FROM node:14-slim # Install latest chrome dev package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others) # Note: this installs the necessary libs to make the bundled version of Chromium that Puppeteer diff --git a/package.json b/package.json index 44c5013af12a7..78017911c238f 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,22 @@ "headless", "automation" ], + "type": "commonjs", "main": "./cjs-entry.js", + "exports": { + ".": { + "import": "./lib/esm/puppeteer/node.js", + "require": "./cjs-entry.js" + }, + "./*": { + "import": "./*", + "require": "./*" + } + }, "types": "lib/types.d.ts", "repository": "github:puppeteer/puppeteer", "engines": { - "node": ">=10.18.1" + "node": ">=14.1.0" }, "scripts": { "test-browser": "wtr", @@ -37,15 +48,18 @@ "doc": "node utils/doclint/cli.js", "generate-api-docs-for-testing": "commonmark docs/api.md > docs/api.html", "clean-lib": "rimraf lib", - "build": "npm run tsc && npm run generate-d-ts", - "tsc": "npm run clean-lib && tsc --version && npm run tsc-cjs && npm run tsc-esm", + "build": "npm run tsc && npm run generate-d-ts && npm run generate-pkg-json", + "tsc": "npm run clean-lib && tsc --version && (npm run tsc-cjs & npm run tsc-esm) && (npm run tsc-compat-cjs & npm run tsc-compat-esm)", "tsc-cjs": "tsc -b src/tsconfig.cjs.json", "tsc-esm": "tsc -b src/tsconfig.esm.json", + "tsc-compat-cjs": "tsc -b compat/cjs/tsconfig.json", + "tsc-compat-esm": "tsc -b compat/esm/tsconfig.json", "apply-next-version": "node utils/apply_next_version.js", "test-install": "scripts/test-install.sh", "clean-docs": "rimraf website/docs && rimraf docs-api-json", "generate-d-ts": "npm run clean-docs && api-extractor run --local --verbose", "generate-docs": "npm run generate-d-ts && api-documenter markdown -i docs-api-json -o website/docs && node utils/remove-tag.js", + "generate-pkg-json": "echo '{\"type\": \"module\"}' > lib/esm/package.json", "ensure-correct-devtools-protocol-revision": "ts-node -s scripts/ensure-correct-devtools-protocol-package", "ensure-pinned-deps": "ts-node -s scripts/ensure-pinned-deps", "test-types-file": "ts-node -s scripts/test-ts-definition-files.ts", @@ -58,6 +72,7 @@ "lib/**/*.d.ts.map", "lib/**/*.js", "lib/**/*.js.map", + "lib/**/package.json", "install.js", "typescript-if-required.js", "cjs-entry.js", @@ -111,7 +126,7 @@ "jpeg-js": "0.4.3", "mime": "3.0.0", "minimist": "1.2.6", - "mocha": "9.2.2", + "mocha": "10.0.0", "ncp": "2.0.0", "pixelmatch": "5.3.0", "pngjs": "6.0.0", diff --git a/scripts/test-install.sh b/scripts/test-install.sh index e59aa39d00be1..70609651f8ec9 100755 --- a/scripts/test-install.sh +++ b/scripts/test-install.sh @@ -1,41 +1,83 @@ #!/usr/bin/env sh set -e +# All tests are headed by a echo 'Test'. +# The general schema is: +# 1. Check we can install from the tarball. +# 2. The install script works and correctly exits without errors +# 3. Requiring/importing Puppeteer from Node works. + +## Puppeter tests + +echo "Setting up Puppeteer tests..." ROOTDIR="$(pwd)" -# Pack the module into a tarball npm pack tarball="$(realpath puppeteer-*.tgz)" + +echo "Testing... Chrome CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR -# Check we can install from the tarball. -# This emulates installing from npm and ensures that: -# 1. we publish the right files in the `files` list from package.json -# 2. The install script works and correctly exits without errors -# 3. Requiring Puppeteer from Node works. npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" +node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');" ls $TMPDIR/node_modules/puppeteer/.local-chromium/ -# Again for Firefox +echo "Testing... Chrome ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer'" +node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';" +node --input-type="module" --eval=" +import puppeteer from 'puppeteer'; +(async () => { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + await page.goto('http://example.com'); + await page.screenshot({ path: 'example.png' }); + await browser.close(); +})(); +" + +echo "Testing... Firefox CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" node --eval="require('puppeteer')" -rm "${tarball}" +node --eval="require('puppeteer/lib/cjs/puppeteer/revisions.js');" ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox -# Again for puppeteer-core +echo "Testing... Firefox ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +PUPPETEER_PRODUCT=firefox npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer'" +node --input-type="module" --eval="import 'puppeteer/lib/esm/puppeteer/revisions.js';" +ls $TMPDIR/node_modules/puppeteer/.local-firefox/linux-*/firefox/firefox + +## Puppeteer Core tests + +echo "Setting up Puppeteer Core tests..." cd $ROOTDIR +rm "${tarball}" node ./utils/prepare_puppeteer_core.js npm pack tarball="$(realpath puppeteer-core-*.tgz)" + +echo "Testing... Puppeteer Core CommonJS" TMPDIR="$(mktemp -d)" cd $TMPDIR -# Check we can install from the tarball. -# This emulates installing from npm and ensures that: -# 1. we publish the right files in the `files` list from package.json -# 2. The install script works and correctly exits without errors -# 3. Requiring Puppeteer Core from Node works. npm install --loglevel silent "${tarball}" node --eval="require('puppeteer-core')" +node --eval="require('puppeteer-core/lib/cjs/puppeteer/revisions.js');" + +echo "Testing... Puppeteer Core ES Modules" +TMPDIR="$(mktemp -d)" +cd $TMPDIR +echo '{"type":"module"}' >>$TMPDIR/package.json +npm install --loglevel silent "${tarball}" +node --input-type="module" --eval="import puppeteer from 'puppeteer-core'" +node --input-type="module" --eval="import 'puppeteer-core/lib/esm/puppeteer/revisions.js';" diff --git a/src/common/Debug.ts b/src/common/Debug.ts index 8ff124b094b3b..105319870462b 100644 --- a/src/common/Debug.ts +++ b/src/common/Debug.ts @@ -54,8 +54,9 @@ import { isNode } from '../environment.js'; */ export const debug = (prefix: string): ((...args: unknown[]) => void) => { if (isNode) { - // eslint-disable-next-line @typescript-eslint/no-var-requires - return require('debug')(prefix); + return async (...logArgs: unknown[]) => { + (await import('debug')).default(prefix)(logArgs); + }; } return (...logArgs: unknown[]): void => { diff --git a/src/compat.ts b/src/compat.ts new file mode 100644 index 0000000000000..873dcea7966e5 --- /dev/null +++ b/src/compat.ts @@ -0,0 +1,3 @@ +declare const puppeteerDirname: string; + +export { puppeteerDirname }; diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000000000..e9104f547d89c --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,8 @@ +import { readFileSync } from 'fs'; +import { dirname, join } from 'path'; +import { puppeteerDirname } from './compat.js'; + +export const rootDirname = dirname(dirname(dirname(puppeteerDirname))); +export const packageVersion = JSON.parse( + readFileSync(join(rootDirname, 'package.json'), { encoding: 'utf8' }) +).version; diff --git a/src/initialize-node.ts b/src/initialize-node.ts index 3d38a47069dc5..9b55b98975fcf 100644 --- a/src/initialize-node.ts +++ b/src/initialize-node.ts @@ -17,24 +17,11 @@ import { PuppeteerNode } from './node/Puppeteer.js'; import { PUPPETEER_REVISIONS } from './revisions.js'; import { sync } from 'pkg-dir'; -import { dirname } from 'path'; import { Product } from './common/Product.js'; - -function resolvePuppeteerRootDirectory(): string | undefined { - try { - // In some environments, like esbuild, this will throw an error. - // We suppress the error since the bundled binary is not expected - // to be used or installed in this case and, therefore, the - // root directory does not have to be known. - return sync(dirname(require.resolve('./initialize-node'))); - } catch (error) { - // Fallback to __dirname. - return sync(__dirname); - } -} +import { rootDirname } from './constants.js'; export const initializePuppeteerNode = (packageName: string): PuppeteerNode => { - const puppeteerRootDirectory = resolvePuppeteerRootDirectory(); + const puppeteerRootDirectory = sync(rootDirname); let preferredRevision = PUPPETEER_REVISIONS.chromium; const isPuppeteerCore = packageName === 'puppeteer-core'; // puppeteer-core ignores environment variables diff --git a/src/node/BrowserFetcher.ts b/src/node/BrowserFetcher.ts index d7a41f049366f..a208e271021e3 100644 --- a/src/node/BrowserFetcher.ts +++ b/src/node/BrowserFetcher.ts @@ -35,6 +35,9 @@ import createHttpsProxyAgent, { import { getProxyForUrl } from 'proxy-from-env'; import { assert } from '../common/assert.js'; +import tar from 'tar-fs'; +import bzip from 'unbzip2-stream'; + const debugFetcher = debug('puppeteer:fetcher'); const downloadURLs = { @@ -499,10 +502,6 @@ function install(archivePath: string, folderPath: string): Promise { * @internal */ function extractTar(tarPath: string, folderPath: string): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const tar = require('tar-fs'); - // eslint-disable-next-line @typescript-eslint/no-var-requires - const bzip = require('unbzip2-stream'); return new Promise((fulfill, reject) => { const tarStream = tar.extract(folderPath); tarStream.on('error', reject); diff --git a/src/node/NodeWebSocketTransport.ts b/src/node/NodeWebSocketTransport.ts index 393763239fef3..57b72509d1942 100644 --- a/src/node/NodeWebSocketTransport.ts +++ b/src/node/NodeWebSocketTransport.ts @@ -13,20 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { ConnectionTransport } from '../common/ConnectionTransport.js'; import NodeWebSocket from 'ws'; +import { ConnectionTransport } from '../common/ConnectionTransport.js'; +import { packageVersion } from '../constants.js'; export class NodeWebSocketTransport implements ConnectionTransport { static create(url: string): Promise { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const pkg = require('../../../../package.json'); return new Promise((resolve, reject) => { const ws = new NodeWebSocket(url, [], { followRedirects: true, perMessageDeflate: false, maxPayload: 256 * 1024 * 1024, // 256Mb headers: { - 'User-Agent': `Puppeteer ${pkg.version}`, + 'User-Agent': `Puppeteer ${packageVersion}`, }, }); diff --git a/utils/prepare_puppeteer_core.js b/utils/prepare_puppeteer_core.js index e1e9a64f2f4ed..a77d7dec66e10 100755 --- a/utils/prepare_puppeteer_core.js +++ b/utils/prepare_puppeteer_core.js @@ -21,7 +21,11 @@ const path = require('path'); const packagePath = path.join(__dirname, '..', 'package.json'); const json = require(packagePath); -json.name = 'puppeteer-core'; delete json.scripts.install; + +json.name = 'puppeteer-core'; json.main = './cjs-entry-core.js'; +json.exports['.'].imports = './lib/esm/puppeteer/node-puppeteer-core.js'; +json.exports['.'].require = './cjs-entry-core.js'; + fs.writeFileSync(packagePath, JSON.stringify(json, null, ' '));