From 25db00fe953453198adb9e1bd71d1bc2a9f04aaa Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 1 Jul 2019 22:17:36 -0700 Subject: [PATCH] worker-farm@1.7.0 --- node_modules/worker-farm/.travis.yml | 4 +- node_modules/worker-farm/README.md | 5 +- node_modules/worker-farm/index.d.ts | 75 ++++++++++++--------- node_modules/worker-farm/lib/child/index.js | 8 ++- node_modules/worker-farm/lib/farm.js | 19 ++++-- node_modules/worker-farm/lib/fork.js | 2 +- node_modules/worker-farm/lib/index.js | 2 +- node_modules/worker-farm/package.json | 32 +++++---- node_modules/worker-farm/tests/index.js | 62 +++++++++++++++-- package-lock.json | 6 +- package.json | 2 +- 11 files changed, 147 insertions(+), 70 deletions(-) diff --git a/node_modules/worker-farm/.travis.yml b/node_modules/worker-farm/.travis.yml index 1c9e2b559d5da..7af56429c010b 100644 --- a/node_modules/worker-farm/.travis.yml +++ b/node_modules/worker-farm/.travis.yml @@ -1,9 +1,9 @@ language: node_js node_js: - - 4 - 6 - 8 - - 9 + - 10 + - 12 branches: only: - master diff --git a/node_modules/worker-farm/README.md b/node_modules/worker-farm/README.md index 982b37cb5dfb9..c52689ed9fc58 100644 --- a/node_modules/worker-farm/README.md +++ b/node_modules/worker-farm/README.md @@ -1,4 +1,4 @@ -# Worker Farm [![Build Status](https://secure.travis-ci.org/rvagg/node-worker-farm.png)](http://travis-ci.org/rvagg/node-worker-farm) +# Worker Farm [![Build Status](https://secure.travis-ci.org/rvagg/node-worker-farm.svg)](http://travis-ci.org/rvagg/node-worker-farm) [![NPM](https://nodei.co/npm/worker-farm.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/worker-farm/) [![NPM](https://nodei.co/npm-dl/worker-farm.png?months=6&height=3)](https://nodei.co/npm/worker-farm/) @@ -111,6 +111,7 @@ If you don't provide an `options` object then the following defaults will be use , maxCallTime : Infinity , maxRetries : Infinity , autoStart : false + , onChild : function() {} } ``` @@ -130,6 +131,8 @@ If you don't provide an `options` object then the following defaults will be use * **autoStart** when set to `true` will start the workers as early as possible. Use this when your workers have to do expensive initialization. That way they'll be ready when the first request comes through. + * **onChild** when new child process starts this callback will be called with subprocess object as an argument. Use this when you need to add some custom communication with child processes. + ### workerFarm.end(farm) Child processes stay alive waiting for jobs indefinitely and your farm manager will stay alive managing its workers, so if you need it to stop then you have to do so explicitly. If you send your farm API to `workerFarm.end()` then it'll cleanly end your worker processes. Note though that it's a *soft* ending so it'll wait for child processes to finish what they are working on before asking them to die. diff --git a/node_modules/worker-farm/index.d.ts b/node_modules/worker-farm/index.d.ts index 682c21f410c51..310e91503fda4 100644 --- a/node_modules/worker-farm/index.d.ts +++ b/node_modules/worker-farm/index.d.ts @@ -1,17 +1,15 @@ -interface Workers { - (callback: WorkerCallback): void; - (arg1: any, callback: WorkerCallback): void; - (arg1: any, arg2: any, callback: WorkerCallback): void; - (arg1: any, arg2: any, arg3: any, callback: WorkerCallback): void; - (arg1: any, arg2: any, arg3: any, arg4: any, callback: WorkerCallback): void; -} +import { ForkOptions } from "child_process"; + +export = Farm; -type WorkerCallback = - | WorkerCallback0 - | WorkerCallback1 - | WorkerCallback2 - | WorkerCallback3 - | WorkerCallback4; +declare function Farm(name: string): Farm.Workers; +declare function Farm(name: string, exportedMethods: string[]): Farm.Workers; +declare function Farm(options: Farm.FarmOptions, name: string): Farm.Workers; +declare function Farm( + options: Farm.FarmOptions, + name: string, + exportedMethods: string[], +): Farm.Workers; type WorkerCallback0 = () => void; type WorkerCallback1 = (arg1: any) => void; @@ -19,26 +17,39 @@ type WorkerCallback2 = (arg1: any, arg2: any) => void; type WorkerCallback3 = (arg1: any, arg2: any, arg3: any) => void; type WorkerCallback4 = (arg1: any, arg2: any, arg3: any, arg4: any) => void; -interface FarmOptions { - maxCallsPerWorker?: number - maxConcurrentWorkers?: number - maxConcurrentCallsPerWorker?: number - maxConcurrentCalls?: number - maxCallTime?: number - maxRetries?: number - autoStart?: boolean -} +declare namespace Farm { + export function end(workers: Workers, callback?: Function): void; -interface WorkerFarm { - (name: string): Workers; - (name: string, exportedMethods: string[]): Workers; - (options: FarmOptions, name: string): Workers; - (options: FarmOptions, name: string, exportedMethods: string[]): Workers; + export interface Workers { + [x: string]: Workers, + (callback: WorkerCallback): void; + (arg1: any, callback: WorkerCallback): void; + (arg1: any, arg2: any, callback: WorkerCallback): void; + (arg1: any, arg2: any, arg3: any, callback: WorkerCallback): void; + ( + arg1: any, + arg2: any, + arg3: any, + arg4: any, + callback: WorkerCallback, + ): void; + } - end: (workers: Workers) => void; -} + export interface FarmOptions { + maxCallsPerWorker?: number; + maxConcurrentWorkers?: number; + maxConcurrentCallsPerWorker?: number; + maxConcurrentCalls?: number; + maxCallTime?: number; + maxRetries?: number; + autoStart?: boolean; + workerOptions?: ForkOptions; + } -declare module "worker-farm" { - const workerFarm: WorkerFarm; - export = workerFarm; + export type WorkerCallback = + | WorkerCallback0 + | WorkerCallback1 + | WorkerCallback2 + | WorkerCallback3 + | WorkerCallback4; } diff --git a/node_modules/worker-farm/lib/child/index.js b/node_modules/worker-farm/lib/child/index.js index f91e08433ab1f..78f63371392af 100644 --- a/node_modules/worker-farm/lib/child/index.js +++ b/node_modules/worker-farm/lib/child/index.js @@ -29,7 +29,7 @@ function handle (data) { _args[0][key] = e[key] }) } - process.send({ idx: idx, child: child, args: _args }) + process.send({ owner: 'farm', idx: idx, child: child, args: _args }) } , exec @@ -46,7 +46,11 @@ function handle (data) { process.on('message', function (data) { + if (data.owner !== 'farm') { + return; + } + if (!$module) return $module = require(data.module) - if (data == 'die') return process.exit(0) + if (data.event == 'die') return process.exit(0) handle(data) }) diff --git a/node_modules/worker-farm/lib/farm.js b/node_modules/worker-farm/lib/farm.js index ef0ab0e1052c7..60720dc561f6c 100644 --- a/node_modules/worker-farm/lib/farm.js +++ b/node_modules/worker-farm/lib/farm.js @@ -10,6 +10,7 @@ const DEFAULT_OPTIONS = { , maxRetries : Infinity , forcedKillTime : 100 , autoStart : false + , onChild : function() {} } const fork = require('./fork') @@ -29,8 +30,8 @@ function Farm (options, path) { Farm.prototype.mkhandle = function (method) { return function () { let args = Array.prototype.slice.call(arguments) - if (this.activeCalls >= this.options.maxConcurrentCalls) { - let err = new MaxConcurrentCallsError('Too many concurrent calls (' + this.activeCalls + ')') + if (this.activeCalls + this.callQueue.length >= this.options.maxConcurrentCalls) { + let err = new MaxConcurrentCallsError('Too many concurrent calls (active: ' + this.activeCalls + ', queued: ' + this.callQueue.length + ')') if (typeof args[args.length - 1] == 'function') return process.nextTick(args[args.length - 1].bind(null, err)) throw err @@ -113,7 +114,14 @@ Farm.prototype.startChild = function () { , exitCode : null } - forked.child.on('message', this.receive.bind(this)) + this.options.onChild(forked.child); + + forked.child.on('message', function(data) { + if (data.owner !== 'farm') { + return; + } + this.receive(data); + }.bind(this)) forked.child.once('exit', function (code) { c.exitCode = code this.onExit(id) @@ -128,7 +136,7 @@ Farm.prototype.startChild = function () { Farm.prototype.stopChild = function (childId) { let child = this.children[childId] if (child) { - child.send('die') + child.send({owner: 'farm', event: 'die'}) setTimeout(function () { if (child.exitCode === null) child.child.kill('SIGKILL') @@ -234,7 +242,8 @@ Farm.prototype.send = function (childId, call) { this.activeCalls++ child.send({ - idx : idx + owner : 'farm' + , idx : idx , child : childId , method : call.method , args : call.args diff --git a/node_modules/worker-farm/lib/fork.js b/node_modules/worker-farm/lib/fork.js index 2843df48474c5..5a035d9749e88 100644 --- a/node_modules/worker-farm/lib/fork.js +++ b/node_modules/worker-farm/lib/fork.js @@ -20,7 +20,7 @@ function fork (forkModule, workerOptions) { // this *should* be picked up by onExit and the operation requeued }) - child.send({ module: forkModule }) + child.send({ owner: 'farm', module: forkModule }) // return a send() function for this child return { diff --git a/node_modules/worker-farm/lib/index.js b/node_modules/worker-farm/lib/index.js index fe574e59b52f7..8c142227355a7 100644 --- a/node_modules/worker-farm/lib/index.js +++ b/node_modules/worker-farm/lib/index.js @@ -26,7 +26,7 @@ function end (api, callback) { for (let i = 0; i < farms.length; i++) if (farms[i] && farms[i].api === api) return farms[i].farm.end(callback) - process.nextTick(callback.bind(null, 'Worker farm not found!')) + process.nextTick(callback.bind(null, new Error('Worker farm not found!'))) } diff --git a/node_modules/worker-farm/package.json b/node_modules/worker-farm/package.json index ba43c44d4bea4..3d6181ae95783 100644 --- a/node_modules/worker-farm/package.json +++ b/node_modules/worker-farm/package.json @@ -1,45 +1,43 @@ { - "_args": [ - [ - "worker-farm@1.6.0", - "/Users/rebecca/code/npm" - ] - ], - "_from": "worker-farm@1.6.0", - "_id": "worker-farm@1.6.0", + "_from": "worker-farm@1.7.0", + "_id": "worker-farm@1.7.0", "_inBundle": false, - "_integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==", + "_integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", "_location": "/worker-farm", "_phantomChildren": {}, "_requested": { "type": "version", "registry": true, - "raw": "worker-farm@1.6.0", + "raw": "worker-farm@1.7.0", "name": "worker-farm", "escapedName": "worker-farm", - "rawSpec": "1.6.0", + "rawSpec": "1.7.0", "saveSpec": null, - "fetchSpec": "1.6.0" + "fetchSpec": "1.7.0" }, "_requiredBy": [ + "#USER", "/", "/libcipm" ], - "_resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz", - "_spec": "1.6.0", - "_where": "/Users/rebecca/code/npm", + "_resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", + "_shasum": "26a94c5391bbca926152002f69b84a4bf772e5a8", + "_spec": "worker-farm@1.7.0", + "_where": "/Users/isaacs/dev/npm/cli", "authors": [ "Rod Vagg @rvagg (https://github.com/rvagg)" ], "bugs": { "url": "https://github.com/rvagg/node-worker-farm/issues" }, + "bundleDependencies": false, "dependencies": { "errno": "~0.1.7" }, + "deprecated": false, "description": "Distribute processing tasks to child processes with an über-simple API and baked-in durability & custom concurrency options.", "devDependencies": { - "tape": "~4.9.0" + "tape": "~4.10.1" }, "homepage": "https://github.com/rvagg/node-worker-farm", "keywords": [ @@ -59,5 +57,5 @@ "test": "node ./tests/" }, "types": "./index.d.ts", - "version": "1.6.0" + "version": "1.7.0" } diff --git a/node_modules/worker-farm/tests/index.js b/node_modules/worker-farm/tests/index.js index ec9deabedda6c..e6be7bb7aabd6 100644 --- a/node_modules/worker-farm/tests/index.js +++ b/node_modules/worker-farm/tests/index.js @@ -51,6 +51,22 @@ tape('simple, exports.fn test', function (t) { }) +tape('on child', function (t) { + t.plan(2) + + let child = workerFarm({ onChild: function(subprocess) { childPid = subprocess.pid } }, childPath) + , childPid = null; + + child(0, function(err, pid) { + t.equal(childPid, pid) + }) + + workerFarm.end(child, function () { + t.ok(true, 'workerFarm ended') + }) +}) + + // use the returned pids to check that we're using a single child process // when maxConcurrentWorkers = 1 tape('single worker', function (t) { @@ -303,10 +319,12 @@ tape('multiple concurrent calls', function (t) { child(defer, function () { if (++cbc == count) { let time = Date.now() - start - // (defer * (count / callsPerWorker + 1)) - if precise it'd be count/callsPerWorker + let min = defer * 1.5 + // (defer * (count / callsPerWorker + 2)) - if precise it'd be count/callsPerWorker // but accounting for IPC and other overhead, we need to give it a bit of extra time, - // hence the +1 - t.ok(time > (defer * 1.5) && time < (defer * (count / callsPerWorker + 1)), 'processed tasks concurrently (' + time + 'ms)') + // hence the +2 + let max = defer * (count / callsPerWorker + 2) + t.ok(time > min && time < max, 'processed tasks concurrently (' + time + ' > ' + min + ' && ' + time + ' < ' + max + ')') workerFarm.end(child, function () { t.ok(true, 'workerFarm ended') }) @@ -474,6 +492,40 @@ tape('test maxConcurrentCalls', function (t) { }) +tape('test maxConcurrentCalls + queue', function (t) { + t.plan(13) + + let child = workerFarm({ maxConcurrentCalls: 4, maxConcurrentWorkers: 2, maxConcurrentCallsPerWorker: 1 }, childPath) + + child(20, function (err) { console.log('ended short1'); t.notOk(err, 'no error, short call 1') }) + child(20, function (err) { console.log('ended short2'); t.notOk(err, 'no error, short call 2') }) + child(300, function (err) { t.notOk(err, 'no error, long call 1') }) + child(300, function (err) { t.notOk(err, 'no error, long call 2') }) + child(20, function (err) { + t.ok(err, 'short call 3 should error') + t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') + }) + child(20, function (err) { + t.ok(err, 'short call 4 should error') + t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') + }) + + // cross fingers and hope the two short jobs have ended + setTimeout(function () { + child(20, function (err) { t.notOk(err, 'no error, delayed short call 1') }) + child(20, function (err) { t.notOk(err, 'no error, delayed short call 2') }) + child(20, function (err) { + t.ok(err, 'delayed short call 3 should error') + t.equal(err.type, 'MaxConcurrentCallsError', 'correct error type') + }) + + workerFarm.end(child, function () { + t.ok(true, 'workerFarm ended') + }) + }, 250) +}) + + // this test should not keep the process running! if the test process // doesn't die then the problem is here tape('test timeout kill', function (t) { @@ -529,12 +581,12 @@ tape('custom arguments can be passed to "fork"', function (t) { let cwd = fs.realpathSync(os.tmpdir()) , workerOptions = { cwd : cwd - , execArgv : ['--no-warnings'] + , execArgv : ['--expose-gc'] } , child = workerFarm({ maxConcurrentWorkers: 1, maxRetries: 5, workerOptions: workerOptions}, childPath, ['args']) child.args(function (err, result) { - t.equal(result.execArgv[0], '--no-warnings', 'flags passed (overridden default)') + t.equal(result.execArgv[0], '--expose-gc', 'flags passed (overridden default)') t.equal(result.cwd, cwd, 'correct cwd folder') }) diff --git a/package-lock.json b/package-lock.json index 386ad3df4f078..1526c51ae5605 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6433,9 +6433,9 @@ "dev": true }, "worker-farm": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz", - "integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", + "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", "requires": { "errno": "~0.1.7" } diff --git a/package.json b/package.json index 3e46b1b1e4aa1..b903d13cc9b94 100644 --- a/package.json +++ b/package.json @@ -141,7 +141,7 @@ "validate-npm-package-license": "^3.0.4", "validate-npm-package-name": "~3.0.0", "which": "^1.3.1", - "worker-farm": "^1.6.0", + "worker-farm": "^1.7.0", "write-file-atomic": "^2.4.3" }, "bundleDependencies": [