Skip to content

Commit

Permalink
feat: better process management (#280)
Browse files Browse the repository at this point in the history 
* feat: add reconnect method

better support for connecting to chrome instances

* v8.9.0-alpha.0

* refactor: improve api

* v9.0.0-alpha.0

* refactor: tweaks

* build(wip): avoid respawn

* v9.0.0-alpha.1

* refactor: better way to respawn browser process

* build: update respawn interface

* v9.0.0-alpha.2

* build: remove duplicate flags

* build: wait until page is closed

* v9.0.0-alpha.3

* build(revert): wait until page is closed

* v9.0.0-alpha.4

* fix: throw an error if browser is not connected

* build: add debug log

* v9.0.0-alpha.5

* fix: await respawn

* build: remove debug logs

* v9.0.0-alpha.6

* fix(revert): await respawn

* v9.0.0-alpha.7

* fix: prevent respawn after respawn

* v9.0.0-alpha.8

* build: add debug log

* v9.0.0-alpha.9

* build: use a different context per session

* v9.0.0-alpha.10

* build: use browser contexts by default

* v9.0.0-alpha.11

* build: update link reference

* fix(function): connect to browser instance properly

* v9.0.0-alpha.12

* test: update tests

* build: createContext returns a promise

* v9.0.0-alpha.13

* build: add respawn debug flag

* v9.0.0-alpha.14

* build: add closePage debug log

* v9.0.0-alpha.15

* fix: avoid finally

`finaly` is executed _after_ try/catch, but we want to cleanup before throw the error

* v9.0.0-alpha.16

* build: log error code

* v9.0.0-alpha.17

* build: remove close lock

* v9.0.0-alpha.18

* fix: destroy context when possible

* v9.0.0-alpha.19

* fix: respawn under EPROTOCOL error

* v9.0.0-alpha.20

* refactor: rewrite lock logic

* v9.0.0-alpha.21

* build: add cookies debug

* fix: remove respawn under EPROTOCOL

* refactor: clean respawn logic

* v9.0.0-alpha.22

* fix: cookie subdomains

* refactor: better cookies parser implementation

* v9.0.0-alpha.23

* refactor: improve cookie debug log

* v9.0.0-alpha.24

* test: adjust settings

* fix: check if browser is closed

* v9.0.0-alpha.25

* fix: createContext returns a promise

* refactor: update cli interface

* build: use once listener
to prevent accumulate listeners over time

* build: update dependencies

* v9.0.0-alpha.26

* build: increment spawn timeout

* v9.0.0-alpha.27

* Revert "build: increment spawn timeout"

This reverts commit c597684.

* v9.0.0-alpha.28

* build: add proxy support per page

* build: add retry & proxy per context

* v9.0.0-alpha.29

* refactor: pass agent instead of proxy

* v9.0.0-alpha.30

* chore: enable lossyDeviceName by default

* docs: add v9 documentation

* v9.0.0-alpha.31

* build: update dependencies

* build: update dependencies

* v9.0.0-rc.0
  • Loading branch information
@Kikobeats
Kikobeats committed Jun 24, 2021
1 parent 756ae42 commit 52f0311
Show file tree
Hide file tree
Showing 41 changed files with 797 additions and 572 deletions.
430 changes: 267 additions & 163 deletions README.md
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion lerna.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"packages": [
"packages/*"
],
"version": "8.8.17",
"version": "9.0.0-rc.0",
"command": {
"bootstrap": {
"npmClientArgs": [
Expand Down
18 changes: 10 additions & 8 deletions packages/browserless/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "browserless",
"description": "The Headless Chrome/Chromium perfomance driver for Node.js",
"homepage": "https://browserless.js.org",
"version": "8.8.17",
"version": "9.0.0-rc.0",
"main": "src/index.js",
"author": {
"email": "josefrancisco.verdu@gmail.com",
Expand Down Expand Up @@ -31,16 +31,17 @@
"text"
],
"dependencies": {
"@browserless/errors": "^8.8.8",
"@browserless/goto": "^8.8.14",
"@browserless/pdf": "^8.8.14",
"@browserless/screenshot": "^8.8.14",
"@browserless/errors": "^9.0.0-alpha.25",
"@browserless/goto": "^9.0.0-rc.0",
"@browserless/pdf": "^9.0.0-rc.0",
"@browserless/screenshot": "^9.0.0-rc.0",
"debug-logfmt": "~1.0.4",
"mutexify": "~1.3.1",
"p-reflect": "~2.1.0",
"p-retry": "~4.5.0",
"p-timeout": "~4.1.0",
"parse-proxy-uri": "~1.0.3",
"pidtree": "~0.5.0",
"puppeteer-proxy": "~2.1.2",
"require-one-of": "~1.0.15"
},
"devDependencies": {
Expand All @@ -55,10 +56,11 @@
"src"
],
"scripts": {
"test": "ava"
"test": "NODE_ENV=test ava"
},
"license": "MIT",
"ava": {
"timeout": "2m"
"timeout": "2m",
"verbose": true
}
}
107 changes: 51 additions & 56 deletions packages/browserless/src/driver.js
View file
Original file line number Diff line number Diff line change
@@ -1,63 +1,61 @@
'use strict'

const debug = require('debug-logfmt')('browserless')
const requireOneOf = require('require-one-of')
const pReflect = require('p-reflect')
const pidtree = require('pidtree')

// flags explained: https://peter.sh/experiments/chromium-command-line-switches/
// default flags: https://github.com/puppeteer/puppeteer/blob/master/lib/Launcher.js#L269
// AWS Lambda flags: https://github.com/alixaxel/chrome-aws-lambda/blob/10feb8d162626d34aad2ee1e657f20956f53fe11/source/index.js
const args = ({ proxy } = {}) =>
[
// base
'--disable-cloud-import',
'--disable-gesture-typing',
'--disable-infobars',
'--disable-notifications',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-offer-upload-credit-cards',
'--disable-print-preview',
'--disable-speech-api',
'--disable-tab-for-desktop-share',
'--disable-translate',
'--disable-voice-input',
'--disable-wake-on-wifi',
'--enable-async-dns',
'--enable-simple-cache-backend',
'--enable-tcp-fast-open',
'--enable-webgl',
'--hide-scrollbars',
'--ignore-gpu-blocklist',
'--mute-audio',
'--no-default-browser-check',
'--no-pings',
'--no-zygote',
'--prerender-from-omnibox=disabled',
'--use-gl=swiftshader',
'--no-sandbox',
// disable navigator.webdriver
/// https://stackoverflow.com/a/60409220
// https://blog.m157q.tw/posts/2020/09/11/bypass-cloudflare-detection-while-using-selenium-with-chromedriver/
'--disable-blink-features=AutomationControlled',
// extra
'--disable-web-security',
'--font-render-hinting=none', // could be 'none', 'medium'
// '--enable-font-antialiasing'
// perf
// '--single-process',
// '--memory-pressure-off',
proxy && `--proxy-server=${proxy.protocol}://${proxy.hostname}:${proxy.port}`
].filter(Boolean)
const defaultArgs = [
// base
'--disable-cloud-import',
'--disable-gesture-typing',
'--disable-infobars',
'--disable-notifications',
'--disable-offer-store-unmasked-wallet-cards',
'--disable-offer-upload-credit-cards',
'--disable-print-preview',
'--disable-speech-api',
'--disable-tab-for-desktop-share',
'--disable-translate',
'--disable-voice-input',
'--disable-wake-on-wifi',
'--enable-async-dns',
'--enable-simple-cache-backend',
'--enable-tcp-fast-open',
'--enable-webgl',
'--ignore-gpu-blocklist',
'--no-default-browser-check',
'--no-pings',
'--no-zygote',
'--prerender-from-omnibox=disabled',
'--use-gl=swiftshader',
'--no-sandbox',
// disable navigator.webdriver
/// https://stackoverflow.com/a/60409220
// https://blog.m157q.tw/posts/2020/09/11/bypass-cloudflare-detection-while-using-selenium-with-chromedriver/
'--disable-blink-features=AutomationControlled',
// extra
'--disable-web-security',
'--font-render-hinting=none' // could be 'none', 'medium'
// '--enable-font-antialiasing'
// perf
// '--single-process',
// '--memory-pressure-off',
]

const spawn = (puppeteer, { mode = 'launch', proxy, ...launchOpts }) =>
puppeteer[mode]({
ignoreHTTPSErrors: true,
// flags explained: https://peter.sh/experiments/chromium-command-line-switches/
// default flags: https://github.com/puppeteer/puppeteer/blob/master/lib/Launcher.js#L269
// AWS Lambda flags: https://github.com/alixaxel/chrome-aws-lambda/blob/10feb8d162626d34aad2ee1e657f20956f53fe11/source/index.js
args: args({ proxy }),
...launchOpts
})
// The param `timeout` means here the maximum time in milliseconds
// to wait for the browser instance to start
const spawn = ({
puppeteer = requireOneOf(['puppeteer', 'puppeteer-core', 'puppeteer-firefox']),
mode = 'launch',
...launchOpts
} = {}) => {
const args = launchOpts.args ? undefined : defaultArgs
return puppeteer[mode]({ ignoreHTTPSErrors: true, timeout: 5000, args, ...launchOpts })
}

const getPid = childProcess => {
if (!childProcess) return null
Expand All @@ -77,23 +75,20 @@ const close = async (childProcess, { signal = 'SIGKILL', ...debugOpts } = {}) =>
if (!pid) return

const pids = await getPids(pid)

pids.forEach(pid => {
try {
process.kill(pid, signal)
} catch (error) {
debug('error', { pid, signal, ...debugOpts, message: error.message || error })
}
} catch (_) {}
})

// It's necessary to call `browser.close` for removing temporal files associated
// and remove listeners attached to the main process
// see https://github.com/puppeteer/puppeteer/blob/main/src/node/BrowserRunner.ts#L129
// see https://github.com/puppeteer/puppeteer/blob/778ac92469d66c542c3c12fe0aa23703dd6315c2/src/node/BrowserRunner.ts#L146
if (childProcess.close) await childProcess.close()

debug('close', { pids, signal, ...debugOpts })

return { pids }
}

module.exports = { getPid, spawn, close, args }
module.exports = { spawn, getPid, close, defaultArgs }

0 comments on commit 52f0311

Please sign in to comment.