From 1cff8fdbb4056d2690b2832e68475230fe07c4e0 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 22 May 2020 18:11:14 -0700 Subject: [PATCH 01/40] lib: initial experimental AbortController implementation AbortController impl based very closely on: https://github.com/mysticatea/abort-controller Marked experimental. Not currently used by any of the existing promise apis. Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/33527 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Matteo Collina Reviewed-By: Anna Henningsen --- .eslintrc.js | 1 + doc/api/cli.md | 8 +++ doc/api/globals.md | 96 +++++++++++++++++++++++++ doc/node.1 | 3 + lib/internal/abort_controller.js | 83 +++++++++++++++++++++ lib/internal/bootstrap/pre_execution.js | 27 +++++++ lib/internal/main/worker_thread.js | 2 + node.gyp | 1 + src/node_options.cc | 4 ++ src/node_options.h | 1 + test/parallel/test-abortcontroller.js | 22 ++++++ tools/doc/type-parser.js | 4 ++ 12 files changed, 252 insertions(+) create mode 100644 lib/internal/abort_controller.js create mode 100644 test/parallel/test-abortcontroller.js diff --git a/.eslintrc.js b/.eslintrc.js index 2ad6cc199cedf7..8464945125b678 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -282,6 +282,7 @@ module.exports = { 'node-core/no-duplicate-requires': 'error', }, globals: { + AbortController: 'readable', Atomics: 'readable', BigInt: 'readable', BigInt64Array: 'readable', diff --git a/doc/api/cli.md b/doc/api/cli.md index 3c39689c620f32..3befa5ae6ddfe6 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -197,6 +197,13 @@ Enable experimental Source Map v3 support for stack traces. Currently, overriding `Error.prepareStackTrace` is ignored when the `--enable-source-maps` flag is set. +### `--experimental-abortcontroller` + + +Enable experimental `AbortController` and `AbortSignal` support. + ### `--experimental-import-meta-resolve` + +> Stability: 1 - Experimental + + + +A utility class used to signal cancelation in selected `Promise`-based APIs. +The API is based on the Web API [`AbortController`][]. + +To use, launch Node.js using the `--experimental-abortcontroller` flag. + +```js +const ac = new AbortController(); + +ac.signal.addEventListener('abort', () => console.log('Aborted!'), + { once: true }); + +ac.abort(); + +console.log(ac.signal.aborted); // Prints True +``` + +### `abortController.abort()` + + +Triggers the abort signal, causing the `abortController.signal` to emit +the `'abort'` event. + +### `abortController.signal` + + +* Type: {AbortSignal} + +### Class: `AbortSignal extends EventTarget` + + +The `AbortSignal` is used to notify observers when the +`abortController.abort()` method is called. + +#### Event: `'abort'` + + +The `'abort'` event is emitted when the `abortController.abort()` method +is called. The callback is invoked with a single object argument with a +single `type` propety set to `'abort'`: + +```js +const ac = new AbortController(); + +// Use either the onabort property... +ac.signal.onabort = () => console.log('aborted!'); + +// Or the EventTarget API... +ac.signal.addEventListener('abort', (event) => { + console.log(event.type); // Prints 'abort' +}, { once: true }); + +ac.abort(); +``` + +The `AbortController` with which the `AbortSignal` is associated will only +ever trigger the `'abort'` event once. Any event listeners attached to the +`AbortSignal` *should* use the `{ once: true }` option (or, if using the +`EventEmitter` APIs to attach a listener, use the `once()` method) to ensure +that the event listener is removed as soon as the `'abort'` event is handled. +Failure to do so may result in memory leaks. + +#### `abortSignal.aborted` + + +* Type: {boolean} True after the `AbortController` has been aborted. + +#### `abortSignal.onabort` + + +* Type: {Function} + +An optional callback function that may be set by user code to be notified +when the `abortController.abort()` function has been called. + ## Class: `Buffer` -* Type: {boolean} Always returns `false`. +* Type: {boolean} True for Node.js internal events, false otherwise. -This is not used in Node.js and is provided purely for completeness. +Currently only `AbortSignal`s' `"abort"` event is fired with `isTrusted` +set to `true`. #### `event.preventDefault()` * `path` {string|Buffer|URL|FileHandle} filename or `FileHandle` * `options` {Object|string} * `encoding` {string|null} **Default:** `null` * `flag` {string} See [support of file system `flags`][]. **Default:** `'r'`. + * `signal` {AbortSignal} allows aborting an in-progress readFile * Returns: {Promise} Asynchronously reads the entire contents of a file. @@ -5432,6 +5460,20 @@ platform-specific. On macOS, Linux, and Windows, the promise will be rejected with an error. On FreeBSD, a representation of the directory's contents will be returned. +It is possible to abort an ongoing `readFile` using an `AbortSignal`. If a +request is aborted the promise returned is rejected with an `AbortError`: + +```js +const controller = new AbortController(); +const signal = controller.signal; +readFile(fileName, { signal }).then((file) => { /* ... */ }); +// Abort the request +controller.abort(); +``` + +Aborting an ongoing request does not abort individual operating +system requests but rather the internal buffering `fs.readFile` performs. + Any specified `FileHandle` has to support reading. ### `fsPromises.readlink(path[, options])` diff --git a/lib/fs.js b/lib/fs.js index 5746fbe1349c74..3aa7a901a562a2 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -316,6 +316,9 @@ function readFile(path, options, callback) { const context = new ReadFileContext(callback, options.encoding); context.isUserFd = isFd(path); // File descriptor ownership + if (options.signal) { + context.signal = options.signal; + } if (context.isUserFd) { process.nextTick(function tick(context) { readFileAfterOpen.call({ context }, null, path); diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 1f81a6f47b1d59..d4e83947feb193 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -30,12 +30,14 @@ const { } = internalBinding('constants').fs; const binding = internalBinding('fs'); const { Buffer } = require('buffer'); + +const { codes, hideStackFrames } = require('internal/errors'); const { ERR_FS_FILE_TOO_LARGE, ERR_INVALID_ARG_TYPE, ERR_INVALID_ARG_VALUE, - ERR_METHOD_NOT_IMPLEMENTED -} = require('internal/errors').codes; + ERR_METHOD_NOT_IMPLEMENTED, +} = codes; const { isArrayBufferView } = require('internal/util/types'); const { rimrafPromises } = require('internal/fs/rimraf'); const { @@ -83,6 +85,13 @@ const { const getDirectoryEntriesPromise = promisify(getDirents); const validateRmOptionsPromise = promisify(validateRmOptions); +let DOMException; +const lazyDOMException = hideStackFrames((message, name) => { + if (DOMException === undefined) + DOMException = internalBinding('messaging').DOMException; + return new DOMException(message, name); +}); + class FileHandle extends JSTransferable { constructor(filehandle) { super(); @@ -260,8 +269,17 @@ async function writeFileHandle(filehandle, data) { } async function readFileHandle(filehandle, options) { + const signal = options && options.signal; + + if (signal && signal.aborted) { + throw lazyDOMException('The operation was aborted', 'AbortError'); + } const statFields = await binding.fstat(filehandle.fd, false, kUsePromises); + if (signal && signal.aborted) { + throw lazyDOMException('The operation was aborted', 'AbortError'); + } + let size; if ((statFields[1/* mode */] & S_IFMT) === S_IFREG) { size = statFields[8/* size */]; @@ -278,6 +296,9 @@ async function readFileHandle(filehandle, options) { MathMin(size, kReadFileMaxChunkSize); let endOfFile = false; do { + if (signal && signal.aborted) { + throw lazyDOMException('The operation was aborted', 'AbortError'); + } const buf = Buffer.alloc(chunkSize); const { bytesRead, buffer } = await read(filehandle, buf, 0, chunkSize, -1); diff --git a/lib/internal/fs/read_file_context.js b/lib/internal/fs/read_file_context.js index 5091a34231665b..faca0e0c331e39 100644 --- a/lib/internal/fs/read_file_context.js +++ b/lib/internal/fs/read_file_context.js @@ -8,6 +8,16 @@ const { Buffer } = require('buffer'); const { FSReqCallback, close, read } = internalBinding('fs'); +const { hideStackFrames } = require('internal/errors'); + + +let DOMException; +const lazyDOMException = hideStackFrames((message, name) => { + if (DOMException === undefined) + DOMException = internalBinding('messaging').DOMException; + return new DOMException(message, name); +}); + // Use 64kb in case the file type is not a regular file and thus do not know the // actual file size. Increasing the value further results in more frequent over // allocation for small files and consumes CPU time and memory that should be @@ -74,6 +84,7 @@ class ReadFileContext { this.pos = 0; this.encoding = encoding; this.err = null; + this.signal = undefined; } read() { @@ -81,6 +92,11 @@ class ReadFileContext { let offset; let length; + if (this.signal && this.signal.aborted) { + return this.close( + lazyDOMException('The operation was aborted', 'AbortError') + ); + } if (this.size === 0) { buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength); offset = 0; diff --git a/lib/internal/fs/utils.js b/lib/internal/fs/utils.js index e3255cdb2f2562..3ff3d9c561675c 100644 --- a/lib/internal/fs/utils.js +++ b/lib/internal/fs/utils.js @@ -37,6 +37,7 @@ const { const { once } = require('internal/util'); const { toPathIfFileURL } = require('internal/url'); const { + validateAbortSignal, validateBoolean, validateInt32, validateUint32 @@ -297,6 +298,10 @@ function getOptions(options, defaultOptions) { if (options.encoding !== 'buffer') assertEncoding(options.encoding); + + if (options.signal !== undefined) { + validateAbortSignal(options.signal, 'options.signal'); + } return options; } diff --git a/test/parallel/test-fs-promises-readfile.js b/test/parallel/test-fs-promises-readfile.js index ff25be75b84ff0..10632ba62e770b 100644 --- a/test/parallel/test-fs-promises-readfile.js +++ b/test/parallel/test-fs-promises-readfile.js @@ -1,3 +1,4 @@ +// Flags: --experimental-abortcontroller 'use strict'; const common = require('../common'); @@ -10,18 +11,21 @@ tmpdir.refresh(); const fn = path.join(tmpdir.path, 'large-file'); -async function validateReadFile() { - // Creating large buffer with random content - const buffer = Buffer.from( - Array.apply(null, { length: 16834 * 2 }) - .map(Math.random) - .map((number) => (number * (1 << 8))) - ); +// Creating large buffer with random content +const largeBuffer = Buffer.from( + Array.apply(null, { length: 16834 * 2 }) + .map(Math.random) + .map((number) => (number * (1 << 8))) +); +async function createLargeFile() { // Writing buffer to a file then try to read it - await writeFile(fn, buffer); + await writeFile(fn, largeBuffer); +} + +async function validateReadFile() { const readBuffer = await readFile(fn); - assert.strictEqual(readBuffer.equals(buffer), true); + assert.strictEqual(readBuffer.equals(largeBuffer), true); } async function validateReadFileProc() { @@ -39,6 +43,28 @@ async function validateReadFileProc() { assert.ok(hostname.length > 0); } -validateReadFile() - .then(() => validateReadFileProc()) - .then(common.mustCall()); +function validateReadFileAbortLogicBefore() { + const controller = new AbortController(); + const signal = controller.signal; + controller.abort(); + assert.rejects(readFile(fn, { signal }), { + name: 'AbortError' + }); +} + +function validateReadFileAbortLogicDuring() { + const controller = new AbortController(); + const signal = controller.signal; + process.nextTick(() => controller.abort()); + assert.rejects(readFile(fn, { signal }), { + name: 'AbortError' + }); +} + +(async () => { + await createLargeFile(); + await validateReadFile(); + await validateReadFileProc(); + await validateReadFileAbortLogicBefore(); + await validateReadFileAbortLogicDuring(); +})().then(common.mustCall()); diff --git a/test/parallel/test-fs-readfile.js b/test/parallel/test-fs-readfile.js index 648bf692d1dcc8..b5cfabc547ad82 100644 --- a/test/parallel/test-fs-readfile.js +++ b/test/parallel/test-fs-readfile.js @@ -1,3 +1,4 @@ +// Flags: --experimental-abortcontroller 'use strict'; const common = require('../common'); @@ -57,3 +58,21 @@ for (const e of fileInfo) { assert.deepStrictEqual(buf, e.contents); })); } +{ + // Test cancellation, before + const controller = new AbortController(); + const signal = controller.signal; + controller.abort(); + fs.readFile(fileInfo[0].name, { signal }, common.mustCall((err, buf) => { + assert.strictEqual(err.name, 'AbortError'); + })); +} +{ + // Test cancellation, during read + const controller = new AbortController(); + const signal = controller.signal; + fs.readFile(fileInfo[0].name, { signal }, common.mustCall((err, buf) => { + assert.strictEqual(err.name, 'AbortError'); + })); + process.nextTick(() => controller.abort()); +} From ceef7e3d5173c24d22d4d3a5d61741f91f84faa3 Mon Sep 17 00:00:00 2001 From: Benjamin Gruenbaum Date: Fri, 6 Nov 2020 13:44:40 +0200 Subject: [PATCH 14/40] fs: support abortsignal in writeFile PR-URL: https://github.com/nodejs/node/pull/35993 Reviewed-By: Antoine du Hamel Reviewed-By: Matteo Collina --- doc/api/fs.md | 61 ++++++++++++++++++++- lib/fs.js | 26 +++++++-- lib/internal/fs/promises.js | 10 +++- test/parallel/test-fs-promises-writefile.js | 12 ++++ test/parallel/test-fs-write-file.js | 30 ++++++++++ 5 files changed, 132 insertions(+), 7 deletions(-) diff --git a/doc/api/fs.md b/doc/api/fs.md index d6d95dd149f9f8..35a0fd462bc652 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -4375,6 +4375,10 @@ details. +* `emitterOrTarget` {EventEmitter|EventTarget} +* `eventName` {string|symbol} +* Returns: {Function[]} + +Returns a copy of the array of listeners for the event named `eventName`. + +For `EventEmitter`s this behaves exactly the same as calling `.listeners` on +the emitter. + +For `EventTarget`s this is the only way to get the event listeners for the +event target. This is useful for debugging and diagnostic purposes. + +```js +const { getEventListeners, EventEmitter } = require('events'); + +{ + const ee = new EventEmitter(); + const listener = () => console.log('Events are fun'); + ee.on('foo', listener); + getEventListeners(ee, 'foo'); // [listener] +} +{ + const et = new EventTarget(); + const listener = () => console.log('Events are fun'); + ee.addEventListener('foo', listener); + getEventListeners(ee, 'foo'); // [listener] +} +``` + ## `events.once(emitter, name[, options])` diff --git a/lib/fs.js b/lib/fs.js index 49dbe2cb621e71..8d27c5b2d9279c 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -1571,6 +1571,17 @@ function watch(filename, options, listener) { if (listener) { watcher.addListener('change', listener); } + if (options.signal) { + if (options.signal.aborted) { + process.nextTick(() => watcher.close()); + } else { + const listener = () => watcher.close(); + options.signal.addEventListener('abort', listener); + watcher.once('close', () => { + options.signal.removeEventListener('abort', listener); + }); + } + } return watcher; } diff --git a/test/parallel/test-fs-watch-abort-signal.js b/test/parallel/test-fs-watch-abort-signal.js new file mode 100644 index 00000000000000..e5589bc1140f8b --- /dev/null +++ b/test/parallel/test-fs-watch-abort-signal.js @@ -0,0 +1,32 @@ +// Flags: --expose-internals --experimental-abortcontroller +'use strict'; + +// Verify that AbortSignal integration works for fs.watch + +const common = require('../common'); + +if (common.isIBMi) + common.skip('IBMi does not support `fs.watch()`'); + +const fs = require('fs'); +const fixtures = require('../common/fixtures'); + + +{ + // Signal aborted after creating the watcher + const file = fixtures.path('empty.js'); + const ac = new AbortController(); + const { signal } = ac; + const watcher = fs.watch(file, { signal }); + watcher.once('close', common.mustCall()); + setImmediate(() => ac.abort()); +} +{ + // Signal aborted before creating the watcher + const file = fixtures.path('empty.js'); + const ac = new AbortController(); + const { signal } = ac; + ac.abort(); + const watcher = fs.watch(file, { signal }); + watcher.once('close', common.mustCall()); +} From 88b3463ebc5af7e0c935af631073efa6a178ee5d Mon Sep 17 00:00:00 2001 From: Nitzan Uziely Date: Tue, 16 Feb 2021 15:18:51 +0200 Subject: [PATCH 33/40] fs: fix pre-aborted writeFile AbortSignal file leak Fix an issue in writeFile where a file is opened, and not closed if the abort signal is aborted after the file was opened but before writing began. PR-URL: https://github.com/nodejs/node/pull/37393 Reviewed-By: Benjamin Gruenbaum Reviewed-By: Antoine du Hamel --- lib/internal/fs/promises.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index a6b50054774f4e..ccac60477ff83c 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -63,6 +63,7 @@ const { const { opendir } = require('internal/fs/dir'); const { parseFileMode, + validateAbortSignal, validateBuffer, validateInteger, validateUint32 @@ -646,14 +647,17 @@ async function writeFile(path, data, options) { data = Buffer.from(data, options.encoding || 'utf8'); } + validateAbortSignal(options.signal); if (path instanceof FileHandle) return writeFileHandle(path, data, options.signal); - const fd = await open(path, flag, options.mode); if (options.signal?.aborted) { throw new lazyDOMException('The operation was aborted', 'AbortError'); } - return PromisePrototypeFinally(writeFileHandle(fd, data), fd.close); + + const fd = await open(path, flag, options.mode); + const { signal } = options; + return PromisePrototypeFinally(writeFileHandle(fd, data, signal), fd.close); } async function appendFile(path, data, options) { @@ -670,6 +674,10 @@ async function readFile(path, options) { if (path instanceof FileHandle) return readFileHandle(path, options); + if (options.signal?.aborted) { + throw lazyDOMException('The operation was aborted', 'AbortError'); + } + const fd = await open(path, flag, 0o666); return PromisePrototypeFinally(readFileHandle(fd, options), fd.close); } From a80d2720b7c6e96151ad69ba475ea287e655b82b Mon Sep 17 00:00:00 2001 From: James M Snell Date: Thu, 11 Mar 2021 07:24:03 -0800 Subject: [PATCH 34/40] doc: recommend checking abortSignal.aborted first Signed-off-by: James M Snell PR-URL: https://github.com/nodejs/node/pull/37714 Reviewed-By: Antoine du Hamel Reviewed-By: Trivikram Kamat Reviewed-By: Darshan Sen Reviewed-By: Rich Trott --- doc/api/globals.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/api/globals.md b/doc/api/globals.md index da86cada230092..e92dd5fc47d732 100644 --- a/doc/api/globals.md +++ b/doc/api/globals.md @@ -89,11 +89,15 @@ ac.abort(); ``` The `AbortController` with which the `AbortSignal` is associated will only -ever trigger the `'abort'` event once. Any event listeners attached to the -`AbortSignal` *should* use the `{ once: true }` option (or, if using the -`EventEmitter` APIs to attach a listener, use the `once()` method) to ensure -that the event listener is removed as soon as the `'abort'` event is handled. -Failure to do so may result in memory leaks. +ever trigger the `'abort'` event once. We recommended that code check +that the `abortSignal.aborted` attribute is `false` before adding an `'abort'` +event listener. + +Any event listeners attached to the `AbortSignal` should use the +`{ once: true }` option (or, if using the `EventEmitter` APIs to attach a +listener, use the `once()` method) to ensure that the event listener is +removed as soon as the `'abort'` event is handled. Failure to do so may +result in memory leaks. #### `abortSignal.aborted` + +* `n` {number} A non-negative number. The maximum number of listeners per + `EventTarget` event. +* `...eventsTargets` {EventTarget[]|EventEmitter[]} Zero or more {EventTarget} + or {EventEmitter} instances. If none are specified, `n` is set as the default + max for all newly created {EventTarget} and {EventEmitter} objects. + +```js +const { + setMaxListeners, + EventEmitter +} = require('events'); + +const target = new EventTarget(); +const emitter = new EventEmitter(); + +setMaxListeners(5, target, emitter); +``` + ### `emitter.addListener(eventName, listener)` + +* Returns: {AbortSignal} + +Returns a new already aborted `AbortSignal`. + #### Event: `'abort'`