From 7ac71f6c4a4586419d7e8ec06c33fec283ce2721 Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Wed, 31 Jul 2019 22:24:12 +0100 Subject: [PATCH 1/6] added support for p-all, properly receives and uses concurrency option, updated readme --- index.js | 33 +++++++++++++++++++++++---------- package.json | 3 ++- readme.md | 6 ++++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 39fabae..7da5c7f 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,8 @@ 'use strict'; const EventEmitter = require('events'); const path = require('path'); +const os = require('os'); +const pAll = require('p-all'); const arrify = require('arrify'); const globby = require('globby'); const cpFile = require('cp-file'); @@ -31,6 +33,15 @@ const preprocessDestinationPath = (source, destination, options) => { module.exports = (source, destination, options = {}) => { const progressEmitter = new EventEmitter(); + let concurrencyDegree; + + if (options.concurrency) { + concurrencyDegree = options.concurrency; + } else { + concurrencyDegree = (os.cpus().length || 1) * 2; + } + + console.log(`Concurrency degree is: ${concurrencyDegree}.`); const promise = (async () => { source = arrify(source); @@ -60,7 +71,7 @@ module.exports = (source, destination, options = {}) => { } const fileProgressHandler = event => { - const fileStatus = copyStatus.get(event.src) || {written: 0, percent: 0}; + const fileStatus = copyStatus.get(event.src) || { written: 0, percent: 0 }; if (fileStatus.written !== event.written || fileStatus.percent !== event.percent) { completedSize -= fileStatus.written; @@ -84,18 +95,20 @@ module.exports = (source, destination, options = {}) => { } }; - return Promise.all(files.map(async sourcePath => { - const from = preprocessSourcePath(sourcePath, options); - const to = preprocessDestinationPath(sourcePath, destination, options); + return pAll(files.map(async sourcePath => { + const operation = async () => { + const from = preprocessSourcePath(sourcePath, options); + const to = preprocessDestinationPath(sourcePath, destination, options); - try { - await cpFile(from, to, options).on('progress', fileProgressHandler); - } catch (error) { - throw new CpyError(`Cannot copy from \`${from}\` to \`${to}\`: ${error.message}`, error); + try { + await cpFile(from, to, options).on('progress', fileProgressHandler); + } catch (error) { + throw new CpyError(`Cannot copy from \`${from}\` to \`${to}\`: ${error.message}`, error); + } } - return to; - })); + return operation; + }), { concurrency: concurrencyDegree }); })(); promise.on = (...arguments_) => { diff --git a/package.json b/package.json index 946ffb7..482fe6a 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "arrify": "^2.0.1", "cp-file": "^7.0.0", "globby": "^9.2.0", - "nested-error-stacks": "^2.1.0" + "nested-error-stacks": "^2.1.0", + "p-all": "^2.1.0" }, "devDependencies": { "ava": "^2.1.0", diff --git a/readme.md b/readme.md index c06a349..ed2387a 100644 --- a/readme.md +++ b/readme.md @@ -84,6 +84,12 @@ Type: `string | Function` Filename or function returning a filename used to rename every file in `source`. +##### concurrency + +Type: `number` + +Number of files being copied concurrently. + ```js const cpy = require('cpy'); From 2f55ac396e5446d71262a4518756868668488086 Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Wed, 31 Jul 2019 23:02:58 +0100 Subject: [PATCH 2/6] updated declarations, cleaner code, improved readme --- index.d.ts | 7 +++++++ index.js | 14 +++++--------- index.test-d.ts | 4 ++++ readme.md | 12 ++++++------ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/index.d.ts b/index.d.ts index e010a37..632eae5 100644 --- a/index.d.ts +++ b/index.d.ts @@ -32,6 +32,13 @@ declare namespace cpy { ``` */ readonly rename?: string | ((basename: string) => string); + + /** + Number of files being copied concurrently. + + @default "(os.cpus().length || 1) * 2" + */ + readonly concurrency?: number; } interface ProgressData { diff --git a/index.js b/index.js index 7da5c7f..c7d5062 100644 --- a/index.js +++ b/index.js @@ -41,8 +41,6 @@ module.exports = (source, destination, options = {}) => { concurrencyDegree = (os.cpus().length || 1) * 2; } - console.log(`Concurrency degree is: ${concurrencyDegree}.`); - const promise = (async () => { source = arrify(source); @@ -71,7 +69,7 @@ module.exports = (source, destination, options = {}) => { } const fileProgressHandler = event => { - const fileStatus = copyStatus.get(event.src) || { written: 0, percent: 0 }; + const fileStatus = copyStatus.get(event.src) || {written: 0, percent: 0}; if (fileStatus.written !== event.written || fileStatus.percent !== event.percent) { completedSize -= fileStatus.written; @@ -95,8 +93,8 @@ module.exports = (source, destination, options = {}) => { } }; - return pAll(files.map(async sourcePath => { - const operation = async () => { + return pAll(files.map(sourcePath => { + return async () => { const from = preprocessSourcePath(sourcePath, options); const to = preprocessDestinationPath(sourcePath, destination, options); @@ -105,10 +103,8 @@ module.exports = (source, destination, options = {}) => { } catch (error) { throw new CpyError(`Cannot copy from \`${from}\` to \`${to}\`: ${error.message}`, error); } - } - - return operation; - }), { concurrency: concurrencyDegree }); + }; + }), {concurrency: concurrencyDegree}); })(); promise.on = (...arguments_) => { diff --git a/index.test-d.ts b/index.test-d.ts index c6a6500..39fec27 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -23,6 +23,10 @@ expectType & ProgressEmitter>( expectType & ProgressEmitter>( cpy('foo.js', 'destination', {overwrite: false}) ); +expectType & ProgressEmitter>( + cpy('foo.js', 'destination', {concurrency: 2}) +); + expectType>( cpy('foo.js', 'destination').on('progress', progress => { diff --git a/readme.md b/readme.md index ed2387a..a2967ad 100644 --- a/readme.md +++ b/readme.md @@ -84,12 +84,6 @@ Type: `string | Function` Filename or function returning a filename used to rename every file in `source`. -##### concurrency - -Type: `number` - -Number of files being copied concurrently. - ```js const cpy = require('cpy'); @@ -100,6 +94,12 @@ const cpy = require('cpy'); })(); ``` +##### concurrency + +Type: `number` +Default: `(os.cpus().length || 1) * 2`; +Number of files being copied concurrently. + ## Progress reporting From 1c884c9628519d454c38a3d332caa9652ea85bca Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Wed, 31 Jul 2019 23:07:26 +0100 Subject: [PATCH 3/6] fixed issue where destination path wasn't being returned --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index c7d5062..f0eab6b 100644 --- a/index.js +++ b/index.js @@ -103,6 +103,8 @@ module.exports = (source, destination, options = {}) => { } catch (error) { throw new CpyError(`Cannot copy from \`${from}\` to \`${to}\`: ${error.message}`, error); } + + return to; }; }), {concurrency: concurrencyDegree}); })(); From 4f4b875074f343018c1a9e4f649a2155e090ab57 Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Wed, 31 Jul 2019 23:34:35 +0100 Subject: [PATCH 4/6] fixed readme --- readme.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index a2967ad..9d15f63 100644 --- a/readme.md +++ b/readme.md @@ -96,8 +96,9 @@ const cpy = require('cpy'); ##### concurrency -Type: `number` -Default: `(os.cpus().length || 1) * 2`; +Type: `number`
+Default: `(os.cpus().length || 1) * 2` + Number of files being copied concurrently. From 7a95e948aa5555666e11dfcda410982691318549 Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Thu, 1 Aug 2019 22:11:54 +0100 Subject: [PATCH 5/6] simplified setting concurrency option default value --- index.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index f0eab6b..dbec438 100644 --- a/index.js +++ b/index.js @@ -33,13 +33,7 @@ const preprocessDestinationPath = (source, destination, options) => { module.exports = (source, destination, options = {}) => { const progressEmitter = new EventEmitter(); - let concurrencyDegree; - - if (options.concurrency) { - concurrencyDegree = options.concurrency; - } else { - concurrencyDegree = (os.cpus().length || 1) * 2; - } + const concurrency = options.concurrency || (os.cpus().length || 1) * 2; const promise = (async () => { source = arrify(source); @@ -106,7 +100,7 @@ module.exports = (source, destination, options = {}) => { return to; }; - }), {concurrency: concurrencyDegree}); + }), {concurrency}); })(); promise.on = (...arguments_) => { From f861103a8536b2b7f38c92fba794e8c4c3555829 Mon Sep 17 00:00:00 2001 From: Joao Silva Date: Fri, 2 Aug 2019 12:32:59 +0100 Subject: [PATCH 6/6] fixed declaration, fixed for passthrough to p-all validation, added test for bad concurrency values --- index.d.ts | 2 +- index.js | 6 ++++-- test.js | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/index.d.ts b/index.d.ts index 632eae5..f4e8c88 100644 --- a/index.d.ts +++ b/index.d.ts @@ -36,7 +36,7 @@ declare namespace cpy { /** Number of files being copied concurrently. - @default "(os.cpus().length || 1) * 2" + @default (os.cpus().length || 1) * 2 */ readonly concurrency?: number; } diff --git a/index.js b/index.js index dbec438..15ce53c 100644 --- a/index.js +++ b/index.js @@ -31,9 +31,11 @@ const preprocessDestinationPath = (source, destination, options) => { return path.join(destination, basename); }; -module.exports = (source, destination, options = {}) => { +module.exports = (source, destination, { + concurrency = (os.cpus().lengh || 1) * 2, + ...options +} = {}) => { const progressEmitter = new EventEmitter(); - const concurrency = options.concurrency || (os.cpus().length || 1) * 2; const promise = (async () => { source = arrify(source); diff --git a/test.js b/test.js index b26d710..dd67d15 100644 --- a/test.js +++ b/test.js @@ -49,6 +49,11 @@ test('copy array of files', async t => { t.is(read('package.json'), read(t.context.tmp, 'package.json')); }); +test('throws on invalid concurrency value', async t => { + await t.throwsAsync(cpy(['license', 'package.json'], t.context.tmp, {concurrency: -2})); + await t.throwsAsync(cpy(['license', 'package.json'], t.context.tmp, {concurrency: 'foo'})); +}); + test('cwd', async t => { fs.mkdirSync(t.context.tmp); fs.mkdirSync(path.join(t.context.tmp, 'cwd'));