Skip to content

Commit

Permalink
fs: add support for async iterators to fsPromises.writeFile
Browse files Browse the repository at this point in the history 
Fixes: #37391

PR-URL: #37490
Backport-PR-URL: #39973
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
  • Loading branch information
@targos
HiroyukiYagihashi authored and targos committed Sep 4, 2021
1 parent 32a5b8f commit cad9d20
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 16 deletions.
6 changes: 5 additions & 1 deletion doc/api/fs.md
View file
Expand Up @@ -1234,6 +1234,9 @@ All the [caveats][] for `fs.watch()` also apply to `fsPromises.watch()`.
<!-- YAML
added: v10.0.0
changes:
- version: REPLACEME
pr-url: https://github.com/nodejs/node/pull/37490
description: The `data` argument supports `AsyncIterable`, `Iterable` & `Stream`.
- version: v14.17.0
pr-url: https://github.com/nodejs/node/pull/35993
description: The options argument may include an AbortSignal to abort an
Expand All @@ -1249,7 +1252,8 @@ changes:
-->
* `file` {string|Buffer|URL|FileHandle} filename or `FileHandle`
* `data` {string|Buffer|Uint8Array|Object}
* `data` {string|Buffer|Uint8Array|Object|AsyncIterable|Iterable
|Stream}
* `options` {Object|string}
* `encoding` {string|null} **Default:** `'utf8'`
* `mode` {integer} **Default:** `0o666`
Expand Down
36 changes: 28 additions & 8 deletions lib/internal/fs/promises.js
View file
Expand Up @@ -25,7 +25,7 @@ const {
const binding = internalBinding('fs');
const { Buffer } = require('buffer');

const { codes, hideStackFrames } = require('internal/errors');
const { AbortError, codes, hideStackFrames } = require('internal/errors');
const {
ERR_FS_FILE_TOO_LARGE,
ERR_INVALID_ARG_TYPE,
Expand Down Expand Up @@ -70,6 +70,7 @@ const {
const pathModule = require('path');
const { promisify } = require('internal/util');
const { watch } = require('internal/fs/watchers');
const { isIterable } = require('internal/streams/utils');

const kHandle = Symbol('kHandle');
const kFd = Symbol('kFd');
Expand Down Expand Up @@ -251,8 +252,23 @@ async function fsCall(fn, handle, ...args) {
}
}

async function writeFileHandle(filehandle, data, signal) {
// `data` could be any kind of typed array.
function checkAborted(signal) {
if (signal && signal.aborted)
throw new AbortError();
}

async function writeFileHandle(filehandle, data, signal, encoding) {
checkAborted(signal);
if (isCustomIterable(data)) {
for await (const buf of data) {
checkAborted(signal);
await write(
filehandle, buf, undefined,
isArrayBufferView(buf) ? buf.length : encoding);
checkAborted(signal);
}
return;
}
data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
let remaining = data.length;
if (remaining === 0) return;
Expand Down Expand Up @@ -422,7 +438,7 @@ async function readv(handle, buffers, position) {
}

async function write(handle, buffer, offset, length, position) {
if (buffer.length === 0)
if (buffer && buffer.length === 0)
return { bytesWritten: 0, buffer };

if (isArrayBufferView(buffer)) {
Expand Down Expand Up @@ -664,22 +680,26 @@ async function writeFile(path, data, options) {
options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'w' });
const flag = options.flag || 'w';

if (!isArrayBufferView(data)) {
if (!isArrayBufferView(data) && !isCustomIterable(data)) {
validateStringAfterArrayBufferView(data, 'data');
data = Buffer.from(data, options.encoding || 'utf8');
}

validateAbortSignal(options.signal);
if (path instanceof FileHandle)
return writeFileHandle(path, data, options.signal);
return writeFileHandle(path, data, options.signal, options.encoding);

if (options.signal?.aborted) {
throw lazyDOMException('The operation was aborted', 'AbortError');
}

const fd = await open(path, flag, options.mode);
const { signal } = options;
return PromisePrototypeFinally(writeFileHandle(fd, data, signal), fd.close);
return PromisePrototypeFinally(
writeFileHandle(fd, data, options.signal, options.encoding), fd.close);
}

function isCustomIterable(obj) {
return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string';
}

async function appendFile(path, data, options) {
Expand Down
2 changes: 1 addition & 1 deletion test/parallel/test-fs-append-file.js
View file
Expand Up @@ -121,7 +121,7 @@ const throwNextTick = (e) => { process.nextTick(() => { throw e; }); };
}

// Test that appendFile does not accept invalid data type (callback API).
[false, 5, {}, [], null, undefined].forEach(async (data) => {
[false, 5, {}, null, undefined].forEach(async (data) => {
const errObj = {
code: 'ERR_INVALID_ARG_TYPE',
message: /"data"|"buffer"/
Expand Down
116 changes: 110 additions & 6 deletions test/parallel/test-fs-promises-writefile.js
View file
Expand Up @@ -8,20 +8,115 @@ const path = require('path');
const tmpdir = require('../common/tmpdir');
const assert = require('assert');
const tmpDir = tmpdir.path;
const { Readable } = require('stream');

tmpdir.refresh();

const dest = path.resolve(tmpDir, 'tmp.txt');
const otherDest = path.resolve(tmpDir, 'tmp-2.txt');
const buffer = Buffer.from('abc'.repeat(1000));
const buffer2 = Buffer.from('xyz'.repeat(1000));
const stream = Readable.from(['a', 'b', 'c']);
const stream2 = Readable.from(['ümlaut', ' ', 'sechzig']);
const iterable = {
expected: 'abc',
*[Symbol.iterator]() {
yield 'a';
yield 'b';
yield 'c';
}
};
function iterableWith(value) {
return {
*[Symbol.iterator]() {
yield value;
}
};
}
const bufferIterable = {
expected: 'abc',
*[Symbol.iterator]() {
yield Buffer.from('a');
yield Buffer.from('b');
yield Buffer.from('c');
}
};
const asyncIterable = {
expected: 'abc',
async* [Symbol.asyncIterator]() {
yield 'a';
yield 'b';
yield 'c';
}
};

async function doWrite() {
await fsPromises.writeFile(dest, buffer);
const data = fs.readFileSync(dest);
assert.deepStrictEqual(data, buffer);
}

async function doWriteStream() {
await fsPromises.writeFile(dest, stream);
const expected = 'abc';
const data = fs.readFileSync(dest, 'utf-8');
assert.deepStrictEqual(data, expected);
}

async function doWriteStreamWithCancel() {
const controller = new AbortController();
const { signal } = controller;
process.nextTick(() => controller.abort());
assert.rejects(fsPromises.writeFile(otherDest, stream, { signal }), {
name: 'AbortError'
});
}

async function doWriteIterable() {
await fsPromises.writeFile(dest, iterable);
const data = fs.readFileSync(dest, 'utf-8');
assert.deepStrictEqual(data, iterable.expected);
}

async function doWriteInvalidIterable() {
await Promise.all(
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
assert.rejects(fsPromises.writeFile(dest, iterableWith(value)), {
code: 'ERR_INVALID_ARG_TYPE',
})
)
);
}

async function doWriteIterableWithEncoding() {
await fsPromises.writeFile(dest, stream2, 'latin1');
const expected = 'ümlaut sechzig';
const data = fs.readFileSync(dest, 'latin1');
assert.deepStrictEqual(data, expected);
}

async function doWriteBufferIterable() {
await fsPromises.writeFile(dest, bufferIterable);
const data = fs.readFileSync(dest, 'utf-8');
assert.deepStrictEqual(data, bufferIterable.expected);
}

async function doWriteAsyncIterable() {
await fsPromises.writeFile(dest, asyncIterable);
const data = fs.readFileSync(dest, 'utf-8');
assert.deepStrictEqual(data, asyncIterable.expected);
}

async function doWriteInvalidValues() {
await Promise.all(
[42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) =>
assert.rejects(fsPromises.writeFile(dest, value), {
code: 'ERR_INVALID_ARG_TYPE',
})
)
);
}

async function doWriteWithCancel() {
const controller = new AbortController();
const { signal } = controller;
Expand Down Expand Up @@ -51,9 +146,18 @@ async function doReadWithEncoding() {
assert.deepStrictEqual(data, syncData);
}

doWrite()
.then(doWriteWithCancel)
.then(doAppend)
.then(doRead)
.then(doReadWithEncoding)
.then(common.mustCall());
(async () => {
await doWrite();
await doWriteWithCancel();
await doAppend();
await doRead();
await doReadWithEncoding();
await doWriteStream();
await doWriteStreamWithCancel();
await doWriteIterable();
await doWriteInvalidIterable();
await doWriteIterableWithEncoding();
await doWriteBufferIterable();
await doWriteAsyncIterable();
await doWriteInvalidValues();
})().then(common.mustCall());

0 comments on commit cad9d20

Please sign in to comment.