From a71df7630e63de98bd88bbbd7171a8ffbd7850ef Mon Sep 17 00:00:00 2001
From: Ouyang Yadong
Date: Tue, 6 Apr 2021 16:09:11 +0800
Subject: [PATCH 001/118] dns: allow `--dns-result-order` to change default dns
verbatim
Allow the `--dns-result-order` option to change the default value
of verbatim in `dns.lookup()`. This is useful when running
Node.js in ipv6-only environments to avoid possible ENETUNREACH
errors.
PR-URL: https://github.com/nodejs/node/pull/38099
Refs: https://github.com/nodejs/node/issues/31566
Reviewed-By: Matteo Collina
Reviewed-By: James M Snell
Reviewed-By: Antoine du Hamel
Reviewed-By: Anna Henningsen
---
doc/api/cli.md | 17 ++++
doc/api/dns.md | 48 +++++++++-
lib/dns.js | 10 +-
lib/internal/dns/promises.js | 8 +-
lib/internal/dns/utils.js | 21 +++++
src/node_options.cc | 7 ++
src/node_options.h | 1 +
.../test-dns-default-verbatim-false.js | 51 ++++++++++
.../test-dns-default-verbatim-true.js | 51 ++++++++++
test/parallel/test-dns-set-default-order.js | 93 +++++++++++++++++++
10 files changed, 298 insertions(+), 9 deletions(-)
create mode 100644 test/parallel/test-dns-default-verbatim-false.js
create mode 100644 test/parallel/test-dns-default-verbatim-true.js
create mode 100644 test/parallel/test-dns-set-default-order.js
diff --git a/doc/api/cli.md b/doc/api/cli.md
index 8fce8ac430da1e..23c54f2e3bfb95 100644
--- a/doc/api/cli.md
+++ b/doc/api/cli.md
@@ -187,6 +187,19 @@ Make built-in language features like `eval` and `new Function` that generate
code from strings throw an exception instead. This does not affect the Node.js
`vm` module.
+### `--dns-result-order=order`
+
+
+Set the default value of `verbatim` in [`dns.lookup()`][] and
+[`dnsPromises.lookup()`][]. The value could be:
+* `ipv4first`: sets default `verbatim` `false`.
+* `verbatim`: sets default `verbatim` `true`.
+
+The default is `ipv4first` and [`dns.setDefaultResultOrder()`][] have higher
+priority than `--dns-result-order`.
+
### `--enable-fips`
+
+* `order` {string} must be `'ipv4first'` or `'verbatim'`.
+
+Set the default value of `verbatim` in [`dns.lookup()`][] and
+[`dnsPromises.lookup()`][]. The value could be:
+* `ipv4first`: sets default `verbatim` `false`.
+* `verbatim`: sets default `verbatim` `true`.
+
+The default is `ipv4first` and [`dns.setDefaultResultOrder()`][] have higher
+priority than [`--dns-result-order`][]. When using [worker threads][],
+[`dns.setDefaultResultOrder()`][] from the main thread won't affect the default
+dns orders in workers.
+
## `dns.setServers(servers)`
+
+* `order` {string} must be `'ipv4first'` or `'verbatim'`.
+
+Set the default value of `verbatim` in [`dns.lookup()`][] and
+[`dnsPromises.lookup()`][]. The value could be:
+* `ipv4first`: sets default `verbatim` `false`.
+* `verbatim`: sets default `verbatim` `true`.
+
+The default is `ipv4first` and [`dnsPromises.setDefaultResultOrder()`][] have
+higher priority than [`--dns-result-order`][]. When using [worker threads][],
+[`dnsPromises.setDefaultResultOrder()`][] from the main thread won't affect the
+default dns orders in workers.
+
### `dnsPromises.setServers(servers)`
+
+## Introduction
+These classes are used to associate state and propagate it throughout
+callbacks and promise chains.
+They allow storing data throughout the lifetime of a web request
+or any other asynchronous duration. It is similar to thread-local storage
+in other languages.
+
+The `AsyncLocalStorage` and `AsyncResource` classes are part of the
+`async_hooks` module:
+
+```js
+const async_hooks = require('async_hooks');
+```
+
+## Class: `AsyncLocalStorage`
+
+
+This class creates stores that stay coherent through asynchronous operations.
+
+While you can create your own implementation on top of the `async_hooks` module,
+`AsyncLocalStorage` should be preferred as it is a performant and memory safe
+implementation that involves significant optimizations that are non-obvious to
+implement.
+
+The following example uses `AsyncLocalStorage` to build a simple logger
+that assigns IDs to incoming HTTP requests and includes them in messages
+logged within each request.
+
+```js
+const http = require('http');
+const { AsyncLocalStorage } = require('async_hooks');
+
+const asyncLocalStorage = new AsyncLocalStorage();
+
+function logWithId(msg) {
+ const id = asyncLocalStorage.getStore();
+ console.log(`${id !== undefined ? id : '-'}:`, msg);
+}
+
+let idSeq = 0;
+http.createServer((req, res) => {
+ asyncLocalStorage.run(idSeq++, () => {
+ logWithId('start');
+ // Imagine any chain of async operations here
+ setImmediate(() => {
+ logWithId('finish');
+ res.end();
+ });
+ });
+}).listen(8080);
+
+http.get('http://localhost:8080');
+http.get('http://localhost:8080');
+// Prints:
+// 0: start
+// 1: start
+// 0: finish
+// 1: finish
+```
+
+Each instance of `AsyncLocalStorage` maintains an independent storage context.
+Multiple instances can safely exist simultaneously without risk of interfering
+with each other data.
+
+### `new AsyncLocalStorage()`
+
+
+Creates a new instance of `AsyncLocalStorage`. Store is only provided within a
+`run()` call or after an `enterWith()` call.
+
+### `asyncLocalStorage.disable()`
+
+
+> Stability: 1 - Experimental
+
+Disables the instance of `AsyncLocalStorage`. All subsequent calls
+to `asyncLocalStorage.getStore()` will return `undefined` until
+`asyncLocalStorage.run()` or `asyncLocalStorage.enterWith()` is called again.
+
+When calling `asyncLocalStorage.disable()`, all current contexts linked to the
+instance will be exited.
+
+Calling `asyncLocalStorage.disable()` is required before the
+`asyncLocalStorage` can be garbage collected. This does not apply to stores
+provided by the `asyncLocalStorage`, as those objects are garbage collected
+along with the corresponding async resources.
+
+Use this method when the `asyncLocalStorage` is not in use anymore
+in the current process.
+
+### `asyncLocalStorage.getStore()`
+
+
+* Returns: {any}
+
+Returns the current store.
+If called outside of an asynchronous context initialized by
+calling `asyncLocalStorage.run()` or `asyncLocalStorage.enterWith()`, it
+returns `undefined`.
+
+### `asyncLocalStorage.enterWith(store)`
+
+
+> Stability: 1 - Experimental
+
+* `store` {any}
+
+Transitions into the context for the remainder of the current
+synchronous execution and then persists the store through any following
+asynchronous calls.
+
+Example:
+
+```js
+const store = { id: 1 };
+// Replaces previous store with the given store object
+asyncLocalStorage.enterWith(store);
+asyncLocalStorage.getStore(); // Returns the store object
+someAsyncOperation(() => {
+ asyncLocalStorage.getStore(); // Returns the same object
+});
+```
+
+This transition will continue for the _entire_ synchronous execution.
+This means that if, for example, the context is entered within an event
+handler subsequent event handlers will also run within that context unless
+specifically bound to another context with an `AsyncResource`. That is why
+`run()` should be preferred over `enterWith()` unless there are strong reasons
+to use the latter method.
+
+```js
+const store = { id: 1 };
+
+emitter.on('my-event', () => {
+ asyncLocalStorage.enterWith(store);
+});
+emitter.on('my-event', () => {
+ asyncLocalStorage.getStore(); // Returns the same object
+});
+
+asyncLocalStorage.getStore(); // Returns undefined
+emitter.emit('my-event');
+asyncLocalStorage.getStore(); // Returns the same object
+```
+
+### `asyncLocalStorage.run(store, callback[, ...args])`
+
+
+* `store` {any}
+* `callback` {Function}
+* `...args` {any}
+
+Runs a function synchronously within a context and returns its
+return value. The store is not accessible outside of the callback function or
+the asynchronous operations created within the callback.
+
+The optional `args` are passed to the callback function.
+
+If the callback function throws an error, the error is thrown by `run()` too.
+The stacktrace is not impacted by this call and the context is exited.
+
+Example:
+
+```js
+const store = { id: 2 };
+try {
+ asyncLocalStorage.run(store, () => {
+ asyncLocalStorage.getStore(); // Returns the store object
+ throw new Error();
+ });
+} catch (e) {
+ asyncLocalStorage.getStore(); // Returns undefined
+ // The error will be caught here
+}
+```
+
+### `asyncLocalStorage.exit(callback[, ...args])`
+
+
+> Stability: 1 - Experimental
+
+* `callback` {Function}
+* `...args` {any}
+
+Runs a function synchronously outside of a context and returns its
+return value. The store is not accessible within the callback function or
+the asynchronous operations created within the callback. Any `getStore()`
+call done within the callback function will always return `undefined`.
+
+The optional `args` are passed to the callback function.
+
+If the callback function throws an error, the error is thrown by `exit()` too.
+The stacktrace is not impacted by this call and the context is re-entered.
+
+Example:
+
+```js
+// Within a call to run
+try {
+ asyncLocalStorage.getStore(); // Returns the store object or value
+ asyncLocalStorage.exit(() => {
+ asyncLocalStorage.getStore(); // Returns undefined
+ throw new Error();
+ });
+} catch (e) {
+ asyncLocalStorage.getStore(); // Returns the same object or value
+ // The error will be caught here
+}
+```
+
+### Usage with `async/await`
+
+If, within an async function, only one `await` call is to run within a context,
+the following pattern should be used:
+
+```js
+async function fn() {
+ await asyncLocalStorage.run(new Map(), () => {
+ asyncLocalStorage.getStore().set('key', value);
+ return foo(); // The return value of foo will be awaited
+ });
+}
+```
+
+In this example, the store is only available in the callback function and the
+functions called by `foo`. Outside of `run`, calling `getStore` will return
+`undefined`.
+
+### Troubleshooting: Context loss
+
+In most cases your application or library code should have no issues with
+`AsyncLocalStorage`. But in rare cases you may face situations when the
+current store is lost in one of the asynchronous operations. In those cases,
+consider the following options.
+
+If your code is callback-based, it is enough to promisify it with
+[`util.promisify()`][], so it starts working with native promises.
+
+If you need to keep using callback-based API, or your code assumes
+a custom thenable implementation, use the [`AsyncResource`][] class
+to associate the asynchronous operation with the correct execution context. To
+do so, you will need to identify the function call responsible for the
+context loss. You can do that by logging the content of
+`asyncLocalStorage.getStore()` after the calls you suspect are responsible for
+the loss. When the code logs `undefined`, the last callback called is probably
+responsible for the context loss.
+
+## Class: `AsyncResource`
+
+
+The class `AsyncResource` is designed to be extended by the embedder's async
+resources. Using this, users can easily trigger the lifetime events of their
+own resources.
+
+The `init` hook will trigger when an `AsyncResource` is instantiated.
+
+The following is an overview of the `AsyncResource` API.
+
+```js
+const { AsyncResource, executionAsyncId } = require('async_hooks');
+
+// AsyncResource() is meant to be extended. Instantiating a
+// new AsyncResource() also triggers init. If triggerAsyncId is omitted then
+// async_hook.executionAsyncId() is used.
+const asyncResource = new AsyncResource(
+ type, { triggerAsyncId: executionAsyncId(), requireManualDestroy: false }
+);
+
+// Run a function in the execution context of the resource. This will
+// * establish the context of the resource
+// * trigger the AsyncHooks before callbacks
+// * call the provided function `fn` with the supplied arguments
+// * trigger the AsyncHooks after callbacks
+// * restore the original execution context
+asyncResource.runInAsyncScope(fn, thisArg, ...args);
+
+// Call AsyncHooks destroy callbacks.
+asyncResource.emitDestroy();
+
+// Return the unique ID assigned to the AsyncResource instance.
+asyncResource.asyncId();
+
+// Return the trigger ID for the AsyncResource instance.
+asyncResource.triggerAsyncId();
+```
+
+### `new AsyncResource(type[, options])`
+
+* `type` {string} The type of async event.
+* `options` {Object}
+ * `triggerAsyncId` {number} The ID of the execution context that created this
+ async event. **Default:** `executionAsyncId()`.
+ * `requireManualDestroy` {boolean} If set to `true`, disables `emitDestroy`
+ when the object is garbage collected. This usually does not need to be set
+ (even if `emitDestroy` is called manually), unless the resource's `asyncId`
+ is retrieved and the sensitive API's `emitDestroy` is called with it.
+ When set to `false`, the `emitDestroy` call on garbage collection
+ will only take place if there is at least one active `destroy` hook.
+ **Default:** `false`.
+
+Example usage:
+
+```js
+class DBQuery extends AsyncResource {
+ constructor(db) {
+ super('DBQuery');
+ this.db = db;
+ }
+
+ getInfo(query, callback) {
+ this.db.get(query, (err, data) => {
+ this.runInAsyncScope(callback, null, err, data);
+ });
+ }
+
+ close() {
+ this.db = null;
+ this.emitDestroy();
+ }
+}
+```
+
+### Static method: `AsyncResource.bind(fn[, type, [thisArg]])`
+
+
+* `fn` {Function} The function to bind to the current execution context.
+* `type` {string} An optional name to associate with the underlying
+ `AsyncResource`.
+* `thisArg` {any}
+
+Binds the given function to the current execution context.
+
+The returned function will have an `asyncResource` property referencing
+the `AsyncResource` to which the function is bound.
+
+### `asyncResource.bind(fn[, thisArg])`
+
+
+* `fn` {Function} The function to bind to the current `AsyncResource`.
+* `thisArg` {any}
+
+Binds the given function to execute to this `AsyncResource`'s scope.
+
+The returned function will have an `asyncResource` property referencing
+the `AsyncResource` to which the function is bound.
+
+### `asyncResource.runInAsyncScope(fn[, thisArg, ...args])`
+
+
+* `fn` {Function} The function to call in the execution context of this async
+ resource.
+* `thisArg` {any} The receiver to be used for the function call.
+* `...args` {any} Optional arguments to pass to the function.
+
+Call the provided function with the provided arguments in the execution context
+of the async resource. This will establish the context, trigger the AsyncHooks
+before callbacks, call the function, trigger the AsyncHooks after callbacks, and
+then restore the original execution context.
+
+### `asyncResource.emitDestroy()`
+
+* Returns: {AsyncResource} A reference to `asyncResource`.
+
+Call all `destroy` hooks. This should only ever be called once. An error will
+be thrown if it is called more than once. This **must** be manually called. If
+the resource is left to be collected by the GC then the `destroy` hooks will
+never be called.
+
+### `asyncResource.asyncId()`
+
+* Returns: {number} The unique `asyncId` assigned to the resource.
+
+### `asyncResource.triggerAsyncId()`
+
+* Returns: {number} The same `triggerAsyncId` that is passed to the
+ `AsyncResource` constructor.
+
+
+### Using `AsyncResource` for a `Worker` thread pool
+
+The following example shows how to use the `AsyncResource` class to properly
+provide async tracking for a [`Worker`][] pool. Other resource pools, such as
+database connection pools, can follow a similar model.
+
+Assuming that the task is adding two numbers, using a file named
+`task_processor.js` with the following content:
+
+```js
+const { parentPort } = require('worker_threads');
+parentPort.on('message', (task) => {
+ parentPort.postMessage(task.a + task.b);
+});
+```
+
+a Worker pool around it could use the following structure:
+
+```js
+const { AsyncResource } = require('async_hooks');
+const { EventEmitter } = require('events');
+const path = require('path');
+const { Worker } = require('worker_threads');
+
+const kTaskInfo = Symbol('kTaskInfo');
+const kWorkerFreedEvent = Symbol('kWorkerFreedEvent');
+
+class WorkerPoolTaskInfo extends AsyncResource {
+ constructor(callback) {
+ super('WorkerPoolTaskInfo');
+ this.callback = callback;
+ }
+
+ done(err, result) {
+ this.runInAsyncScope(this.callback, null, err, result);
+ this.emitDestroy(); // `TaskInfo`s are used only once.
+ }
+}
+
+class WorkerPool extends EventEmitter {
+ constructor(numThreads) {
+ super();
+ this.numThreads = numThreads;
+ this.workers = [];
+ this.freeWorkers = [];
+ this.tasks = [];
+
+ for (let i = 0; i < numThreads; i++)
+ this.addNewWorker();
+
+ // Any time the kWorkerFreedEvent is emitted, dispatch
+ // the next task pending in the queue, if any.
+ this.on(kWorkerFreedEvent, () => {
+ if (this.tasks.length > 0) {
+ const { task, callback } = this.tasks.shift();
+ this.runTask(task, callback);
+ }
+ });
+ }
+
+ addNewWorker() {
+ const worker = new Worker(path.resolve(__dirname, 'task_processor.js'));
+ worker.on('message', (result) => {
+ // In case of success: Call the callback that was passed to `runTask`,
+ // remove the `TaskInfo` associated with the Worker, and mark it as free
+ // again.
+ worker[kTaskInfo].done(null, result);
+ worker[kTaskInfo] = null;
+ this.freeWorkers.push(worker);
+ this.emit(kWorkerFreedEvent);
+ });
+ worker.on('error', (err) => {
+ // In case of an uncaught exception: Call the callback that was passed to
+ // `runTask` with the error.
+ if (worker[kTaskInfo])
+ worker[kTaskInfo].done(err, null);
+ else
+ this.emit('error', err);
+ // Remove the worker from the list and start a new Worker to replace the
+ // current one.
+ this.workers.splice(this.workers.indexOf(worker), 1);
+ this.addNewWorker();
+ });
+ this.workers.push(worker);
+ this.freeWorkers.push(worker);
+ this.emit(kWorkerFreedEvent);
+ }
+
+ runTask(task, callback) {
+ if (this.freeWorkers.length === 0) {
+ // No free threads, wait until a worker thread becomes free.
+ this.tasks.push({ task, callback });
+ return;
+ }
+
+ const worker = this.freeWorkers.pop();
+ worker[kTaskInfo] = new WorkerPoolTaskInfo(callback);
+ worker.postMessage(task);
+ }
+
+ close() {
+ for (const worker of this.workers) worker.terminate();
+ }
+}
+
+module.exports = WorkerPool;
+```
+
+Without the explicit tracking added by the `WorkerPoolTaskInfo` objects,
+it would appear that the callbacks are associated with the individual `Worker`
+objects. However, the creation of the `Worker`s is not associated with the
+creation of the tasks and does not provide information about when tasks
+were scheduled.
+
+This pool could be used as follows:
+
+```js
+const WorkerPool = require('./worker_pool.js');
+const os = require('os');
+
+const pool = new WorkerPool(os.cpus().length);
+
+let finished = 0;
+for (let i = 0; i < 10; i++) {
+ pool.runTask({ a: 42, b: 100 }, (err, result) => {
+ console.log(i, err, result);
+ if (++finished === 10)
+ pool.close();
+ });
+}
+```
+
+### Integrating `AsyncResource` with `EventEmitter`
+
+Event listeners triggered by an [`EventEmitter`][] may be run in a different
+execution context than the one that was active when `eventEmitter.on()` was
+called.
+
+The following example shows how to use the `AsyncResource` class to properly
+associate an event listener with the correct execution context. The same
+approach can be applied to a [`Stream`][] or a similar event-driven class.
+
+```js
+const { createServer } = require('http');
+const { AsyncResource, executionAsyncId } = require('async_hooks');
+
+const server = createServer((req, res) => {
+ req.on('close', AsyncResource.bind(() => {
+ // Execution context is bound to the current outer scope.
+ }));
+ req.on('close', () => {
+ // Execution context is bound to the scope that caused 'close' to emit.
+ });
+ res.end();
+}).listen(3000);
+```
+[`AsyncResource`]: #async_context_class_asyncresource
+[`EventEmitter`]: events.md#events_class_eventemitter
+[`Stream`]: stream.md#stream_stream
+[`Worker`]: worker_threads.md#worker_threads_class_worker
+[`util.promisify()`]: util.md#util_util_promisify_original
diff --git a/doc/api/async_hooks.md b/doc/api/async_hooks.md
index 55cc0542561334..f6805102fdf050 100644
--- a/doc/api/async_hooks.md
+++ b/doc/api/async_hooks.md
@@ -664,575 +664,20 @@ like I/O, connection pooling, or managing callback queues may use the
### Class: `AsyncResource`
-The class `AsyncResource` is designed to be extended by the embedder's async
-resources. Using this, users can easily trigger the lifetime events of their
-own resources.
-
-The `init` hook will trigger when an `AsyncResource` is instantiated.
-
-The following is an overview of the `AsyncResource` API.
-
-```js
-const { AsyncResource, executionAsyncId } = require('async_hooks');
-
-// AsyncResource() is meant to be extended. Instantiating a
-// new AsyncResource() also triggers init. If triggerAsyncId is omitted then
-// async_hook.executionAsyncId() is used.
-const asyncResource = new AsyncResource(
- type, { triggerAsyncId: executionAsyncId(), requireManualDestroy: false }
-);
-
-// Run a function in the execution context of the resource. This will
-// * establish the context of the resource
-// * trigger the AsyncHooks before callbacks
-// * call the provided function `fn` with the supplied arguments
-// * trigger the AsyncHooks after callbacks
-// * restore the original execution context
-asyncResource.runInAsyncScope(fn, thisArg, ...args);
-
-// Call AsyncHooks destroy callbacks.
-asyncResource.emitDestroy();
-
-// Return the unique ID assigned to the AsyncResource instance.
-asyncResource.asyncId();
-
-// Return the trigger ID for the AsyncResource instance.
-asyncResource.triggerAsyncId();
-```
-
-#### `new AsyncResource(type[, options])`
-
-* `type` {string} The type of async event.
-* `options` {Object}
- * `triggerAsyncId` {number} The ID of the execution context that created this
- async event. **Default:** `executionAsyncId()`.
- * `requireManualDestroy` {boolean} If set to `true`, disables `emitDestroy`
- when the object is garbage collected. This usually does not need to be set
- (even if `emitDestroy` is called manually), unless the resource's `asyncId`
- is retrieved and the sensitive API's `emitDestroy` is called with it.
- When set to `false`, the `emitDestroy` call on garbage collection
- will only take place if there is at least one active `destroy` hook.
- **Default:** `false`.
-
-Example usage:
-
-```js
-class DBQuery extends AsyncResource {
- constructor(db) {
- super('DBQuery');
- this.db = db;
- }
-
- getInfo(query, callback) {
- this.db.get(query, (err, data) => {
- this.runInAsyncScope(callback, null, err, data);
- });
- }
-
- close() {
- this.db = null;
- this.emitDestroy();
- }
-}
-```
-
-#### Static method: `AsyncResource.bind(fn[, type, [thisArg]])`
-
-
-* `fn` {Function} The function to bind to the current execution context.
-* `type` {string} An optional name to associate with the underlying
- `AsyncResource`.
-* `thisArg` {any}
-
-Binds the given function to the current execution context.
-
-The returned function will have an `asyncResource` property referencing
-the `AsyncResource` to which the function is bound.
-
-#### `asyncResource.bind(fn[, thisArg])`
-
-
-* `fn` {Function} The function to bind to the current `AsyncResource`.
-* `thisArg` {any}
-
-Binds the given function to execute to this `AsyncResource`'s scope.
-
-The returned function will have an `asyncResource` property referencing
-the `AsyncResource` to which the function is bound.
-
-#### `asyncResource.runInAsyncScope(fn[, thisArg, ...args])`
-
-
-* `fn` {Function} The function to call in the execution context of this async
- resource.
-* `thisArg` {any} The receiver to be used for the function call.
-* `...args` {any} Optional arguments to pass to the function.
-
-Call the provided function with the provided arguments in the execution context
-of the async resource. This will establish the context, trigger the AsyncHooks
-before callbacks, call the function, trigger the AsyncHooks after callbacks, and
-then restore the original execution context.
-
-#### `asyncResource.emitDestroy()`
-
-* Returns: {AsyncResource} A reference to `asyncResource`.
-
-Call all `destroy` hooks. This should only ever be called once. An error will
-be thrown if it is called more than once. This **must** be manually called. If
-the resource is left to be collected by the GC then the `destroy` hooks will
-never be called.
-
-#### `asyncResource.asyncId()`
-
-* Returns: {number} The unique `asyncId` assigned to the resource.
-
-#### `asyncResource.triggerAsyncId()`
-
-* Returns: {number} The same `triggerAsyncId` that is passed to the
- `AsyncResource` constructor.
-
-
-### Using `AsyncResource` for a `Worker` thread pool
-
-The following example shows how to use the `AsyncResource` class to properly
-provide async tracking for a [`Worker`][] pool. Other resource pools, such as
-database connection pools, can follow a similar model.
-
-Assuming that the task is adding two numbers, using a file named
-`task_processor.js` with the following content:
-
-```js
-const { parentPort } = require('worker_threads');
-parentPort.on('message', (task) => {
- parentPort.postMessage(task.a + task.b);
-});
-```
-
-a Worker pool around it could use the following structure:
-
-```js
-const { AsyncResource } = require('async_hooks');
-const { EventEmitter } = require('events');
-const path = require('path');
-const { Worker } = require('worker_threads');
-
-const kTaskInfo = Symbol('kTaskInfo');
-const kWorkerFreedEvent = Symbol('kWorkerFreedEvent');
-
-class WorkerPoolTaskInfo extends AsyncResource {
- constructor(callback) {
- super('WorkerPoolTaskInfo');
- this.callback = callback;
- }
-
- done(err, result) {
- this.runInAsyncScope(this.callback, null, err, result);
- this.emitDestroy(); // `TaskInfo`s are used only once.
- }
-}
-
-class WorkerPool extends EventEmitter {
- constructor(numThreads) {
- super();
- this.numThreads = numThreads;
- this.workers = [];
- this.freeWorkers = [];
- this.tasks = [];
-
- for (let i = 0; i < numThreads; i++)
- this.addNewWorker();
-
- // Any time the kWorkerFreedEvent is emitted, dispatch
- // the next task pending in the queue, if any.
- this.on(kWorkerFreedEvent, () => {
- if (this.tasks.length > 0) {
- const { task, callback } = this.tasks.shift();
- this.runTask(task, callback);
- }
- });
- }
-
- addNewWorker() {
- const worker = new Worker(path.resolve(__dirname, 'task_processor.js'));
- worker.on('message', (result) => {
- // In case of success: Call the callback that was passed to `runTask`,
- // remove the `TaskInfo` associated with the Worker, and mark it as free
- // again.
- worker[kTaskInfo].done(null, result);
- worker[kTaskInfo] = null;
- this.freeWorkers.push(worker);
- this.emit(kWorkerFreedEvent);
- });
- worker.on('error', (err) => {
- // In case of an uncaught exception: Call the callback that was passed to
- // `runTask` with the error.
- if (worker[kTaskInfo])
- worker[kTaskInfo].done(err, null);
- else
- this.emit('error', err);
- // Remove the worker from the list and start a new Worker to replace the
- // current one.
- this.workers.splice(this.workers.indexOf(worker), 1);
- this.addNewWorker();
- });
- this.workers.push(worker);
- this.freeWorkers.push(worker);
- this.emit(kWorkerFreedEvent);
- }
-
- runTask(task, callback) {
- if (this.freeWorkers.length === 0) {
- // No free threads, wait until a worker thread becomes free.
- this.tasks.push({ task, callback });
- return;
- }
-
- const worker = this.freeWorkers.pop();
- worker[kTaskInfo] = new WorkerPoolTaskInfo(callback);
- worker.postMessage(task);
- }
-
- close() {
- for (const worker of this.workers) worker.terminate();
- }
-}
-
-module.exports = WorkerPool;
-```
-
-Without the explicit tracking added by the `WorkerPoolTaskInfo` objects,
-it would appear that the callbacks are associated with the individual `Worker`
-objects. However, the creation of the `Worker`s is not associated with the
-creation of the tasks and does not provide information about when tasks
-were scheduled.
-
-This pool could be used as follows:
-
-```js
-const WorkerPool = require('./worker_pool.js');
-const os = require('os');
-
-const pool = new WorkerPool(os.cpus().length);
-
-let finished = 0;
-for (let i = 0; i < 10; i++) {
- pool.runTask({ a: 42, b: 100 }, (err, result) => {
- console.log(i, err, result);
- if (++finished === 10)
- pool.close();
- });
-}
-```
-
-### Integrating `AsyncResource` with `EventEmitter`
-
-Event listeners triggered by an [`EventEmitter`][] may be run in a different
-execution context than the one that was active when `eventEmitter.on()` was
-called.
-
-The following example shows how to use the `AsyncResource` class to properly
-associate an event listener with the correct execution context. The same
-approach can be applied to a [`Stream`][] or a similar event-driven class.
-
-```js
-const { createServer } = require('http');
-const { AsyncResource, executionAsyncId } = require('async_hooks');
-
-const server = createServer((req, res) => {
- req.on('close', AsyncResource.bind(() => {
- // Execution context is bound to the current outer scope.
- }));
- req.on('close', () => {
- // Execution context is bound to the scope that caused 'close' to emit.
- });
- res.end();
-}).listen(3000);
-```
+The documentation for this class has moved [`AsyncResource`][].
## Class: `AsyncLocalStorage`
-
-
-This class is used to create asynchronous state within callbacks and promise
-chains. It allows storing data throughout the lifetime of a web request
-or any other asynchronous duration. It is similar to thread-local storage
-in other languages.
-
-While you can create your own implementation on top of the `async_hooks` module,
-`AsyncLocalStorage` should be preferred as it is a performant and memory safe
-implementation that involves significant optimizations that are non-obvious to
-implement.
-
-The following example uses `AsyncLocalStorage` to build a simple logger
-that assigns IDs to incoming HTTP requests and includes them in messages
-logged within each request.
-
-```js
-const http = require('http');
-const { AsyncLocalStorage } = require('async_hooks');
-
-const asyncLocalStorage = new AsyncLocalStorage();
-
-function logWithId(msg) {
- const id = asyncLocalStorage.getStore();
- console.log(`${id !== undefined ? id : '-'}:`, msg);
-}
-
-let idSeq = 0;
-http.createServer((req, res) => {
- asyncLocalStorage.run(idSeq++, () => {
- logWithId('start');
- // Imagine any chain of async operations here
- setImmediate(() => {
- logWithId('finish');
- res.end();
- });
- });
-}).listen(8080);
-
-http.get('http://localhost:8080');
-http.get('http://localhost:8080');
-// Prints:
-// 0: start
-// 1: start
-// 0: finish
-// 1: finish
-```
-
-When having multiple instances of `AsyncLocalStorage`, they are independent
-from each other. It is safe to instantiate this class multiple times.
-
-### `new AsyncLocalStorage()`
-
-
-Creates a new instance of `AsyncLocalStorage`. Store is only provided within a
-`run()` call or after an `enterWith()` call.
-
-### `asyncLocalStorage.disable()`
-
-
-Disables the instance of `AsyncLocalStorage`. All subsequent calls
-to `asyncLocalStorage.getStore()` will return `undefined` until
-`asyncLocalStorage.run()` or `asyncLocalStorage.enterWith()` is called again.
-
-When calling `asyncLocalStorage.disable()`, all current contexts linked to the
-instance will be exited.
-
-Calling `asyncLocalStorage.disable()` is required before the
-`asyncLocalStorage` can be garbage collected. This does not apply to stores
-provided by the `asyncLocalStorage`, as those objects are garbage collected
-along with the corresponding async resources.
-
-Use this method when the `asyncLocalStorage` is not in use anymore
-in the current process.
-
-### `asyncLocalStorage.getStore()`
-
-
-* Returns: {any}
-
-Returns the current store.
-If called outside of an asynchronous context initialized by
-calling `asyncLocalStorage.run()` or `asyncLocalStorage.enterWith()`, it
-returns `undefined`.
-
-### `asyncLocalStorage.enterWith(store)`
-
-
-* `store` {any}
-
-Transitions into the context for the remainder of the current
-synchronous execution and then persists the store through any following
-asynchronous calls.
-
-Example:
-
-```js
-const store = { id: 1 };
-// Replaces previous store with the given store object
-asyncLocalStorage.enterWith(store);
-asyncLocalStorage.getStore(); // Returns the store object
-someAsyncOperation(() => {
- asyncLocalStorage.getStore(); // Returns the same object
-});
-```
-
-This transition will continue for the _entire_ synchronous execution.
-This means that if, for example, the context is entered within an event
-handler subsequent event handlers will also run within that context unless
-specifically bound to another context with an `AsyncResource`. That is why
-`run()` should be preferred over `enterWith()` unless there are strong reasons
-to use the latter method.
-
-```js
-const store = { id: 1 };
-
-emitter.on('my-event', () => {
- asyncLocalStorage.enterWith(store);
-});
-emitter.on('my-event', () => {
- asyncLocalStorage.getStore(); // Returns the same object
-});
-
-asyncLocalStorage.getStore(); // Returns undefined
-emitter.emit('my-event');
-asyncLocalStorage.getStore(); // Returns the same object
-```
-
-### `asyncLocalStorage.run(store, callback[, ...args])`
-
-
-* `store` {any}
-* `callback` {Function}
-* `...args` {any}
-
-Runs a function synchronously within a context and returns its
-return value. The store is not accessible outside of the callback function.
-The store is accessible to any asynchronous operations created within the
-callback.
-
-The optional `args` are passed to the callback function.
-
-If the callback function throws an error, the error is thrown by `run()` too.
-The stacktrace is not impacted by this call and the context is exited.
-
-Example:
-
-```js
-const store = { id: 2 };
-try {
- asyncLocalStorage.run(store, () => {
- asyncLocalStorage.getStore(); // Returns the store object
- setTimeout(() => {
- asyncLocalStorage.getStore(); // Returns the store object
- }, 200);
- throw new Error();
- });
-} catch (e) {
- asyncLocalStorage.getStore(); // Returns undefined
- // The error will be caught here
-}
-```
-
-### `asyncLocalStorage.exit(callback[, ...args])`
-
-
-* `callback` {Function}
-* `...args` {any}
-
-Runs a function synchronously outside of a context and returns its
-return value. The store is not accessible within the callback function or
-the asynchronous operations created within the callback. Any `getStore()`
-call done within the callback function will always return `undefined`.
-
-The optional `args` are passed to the callback function.
-
-If the callback function throws an error, the error is thrown by `exit()` too.
-The stacktrace is not impacted by this call and the context is re-entered.
-
-Example:
-
-```js
-// Within a call to run
-try {
- asyncLocalStorage.getStore(); // Returns the store object or value
- asyncLocalStorage.exit(() => {
- asyncLocalStorage.getStore(); // Returns undefined
- throw new Error();
- });
-} catch (e) {
- asyncLocalStorage.getStore(); // Returns the same object or value
- // The error will be caught here
-}
-```
-
-### Usage with `async/await`
-
-If, within an async function, only one `await` call is to run within a context,
-the following pattern should be used:
-
-```js
-async function fn() {
- await asyncLocalStorage.run(new Map(), () => {
- asyncLocalStorage.getStore().set('key', value);
- return foo(); // The return value of foo will be awaited
- });
-}
-```
-
-In this example, the store is only available in the callback function and the
-functions called by `foo`. Outside of `run`, calling `getStore` will return
-`undefined`.
-
-### Troubleshooting
-
-In most cases your application or library code should have no issues with
-`AsyncLocalStorage`. But in rare cases you may face situations when the
-current store is lost in one of asynchronous operations. In those cases,
-consider the following options.
-
-If your code is callback-based, it is enough to promisify it with
-[`util.promisify()`][], so it starts working with native promises.
-If you need to keep using callback-based API, or your code assumes
-a custom thenable implementation, use the [`AsyncResource`][] class
-to associate the asynchronous operation with the correct execution context.
+The documentation for this class has moved [`AsyncLocalStorage`][].
[Hook Callbacks]: #async_hooks_hook_callbacks
[PromiseHooks]: https://docs.google.com/document/d/1rda3yKGHimKIhg5YeoAmCOtyURgsbTH_qaYR79FELlk/edit
-[`AsyncResource`]: #async_hooks_class_asyncresource
+[`AsyncLocalStorage`]: async_context.md#async_context_class_asynclocalstorage
+[`AsyncResource`]: async_context.md#async_context_class_asyncresource
[`after` callback]: #async_hooks_after_asyncid
[`before` callback]: #async_hooks_before_asyncid
[`destroy` callback]: #async_hooks_destroy_asyncid
[`init` callback]: #async_hooks_init_asyncid_type_triggerasyncid_resource
[`promiseResolve` callback]: #async_hooks_promiseresolve_asyncid
-[`EventEmitter`]: events.md#events_class_eventemitter
-[`Stream`]: stream.md#stream_stream
[`Worker`]: worker_threads.md#worker_threads_class_worker
-[`util.promisify()`]: util.md#util_util_promisify_original
[promise execution tracking]: #async_hooks_promise_execution_tracking
diff --git a/doc/api/deprecations.md b/doc/api/deprecations.md
index cc6b61206c5efe..c771baa525f308 100644
--- a/doc/api/deprecations.md
+++ b/doc/api/deprecations.md
@@ -2802,7 +2802,7 @@ deprecated and should no longer be used.
[`SlowBuffer`]: buffer.md#buffer_class_slowbuffer
[`WriteStream.open()`]: fs.md#fs_class_fs_writestream
[`assert`]: assert.md
-[`asyncResource.runInAsyncScope()`]: async_hooks.md#async_hooks_asyncresource_runinasyncscope_fn_thisarg_args
+[`asyncResource.runInAsyncScope()`]: async_context.md#async_context_asyncresource_runinasyncscope_fn_thisarg_args
[`child_process`]: child_process.md
[`clearInterval()`]: timers.md#timers_clearinterval_timeout
[`clearTimeout()`]: timers.md#timers_cleartimeout_timeout
diff --git a/doc/api/index.md b/doc/api/index.md
index a17fb4b666c7d4..f45aa17ecc451a 100644
--- a/doc/api/index.md
+++ b/doc/api/index.md
@@ -11,6 +11,7 @@
* [Assertion testing](assert.md)
+* [Async_context](async_context.md)
* [Async hooks](async_hooks.md)
* [Buffer](buffer.md)
* [C++ addons](addons.md)
From 74f5e30d692e9559ee2d7521c9feddf55f941cc7 Mon Sep 17 00:00:00 2001
From: legendecas
Date: Mon, 24 May 2021 23:39:36 +0800
Subject: [PATCH 016/118] node-api: rtn pending excep on napi_new_instance
When there are any JavaScript exceptions pending,
`napi_pending_exception` should be returned.
PR-URL: https://github.com/nodejs/node/pull/38798
Reviewed-By: Anna Henningsen
Reviewed-By: Michael Dawson
---
src/js_native_api_v8.cc | 2 +-
test/js-native-api/test_exception/test.js | 47 +++++++++++++++++++
.../test_exception/test_exception.c | 31 ++++++++++++
3 files changed, 79 insertions(+), 1 deletion(-)
diff --git a/src/js_native_api_v8.cc b/src/js_native_api_v8.cc
index 1541f7eb36ea43..95f8212d870a72 100644
--- a/src/js_native_api_v8.cc
+++ b/src/js_native_api_v8.cc
@@ -2660,7 +2660,7 @@ napi_status napi_new_instance(napi_env env,
auto maybe = ctor->NewInstance(context, argc,
reinterpret_cast*>(const_cast(argv)));
- CHECK_MAYBE_EMPTY(env, maybe, napi_generic_failure);
+ CHECK_MAYBE_EMPTY(env, maybe, napi_pending_exception);
*result = v8impl::JsValueFromV8LocalValue(maybe.ToLocalChecked());
return GET_RETURN_STATUS(env);
diff --git a/test/js-native-api/test_exception/test.js b/test/js-native-api/test_exception/test.js
index ff11b702198b88..38e8fd1d6b6bdc 100644
--- a/test/js-native-api/test_exception/test.js
+++ b/test/js-native-api/test_exception/test.js
@@ -48,6 +48,34 @@ const test_exception = (function() {
` thrown, but ${returnedError} was passed`);
}
+
+{
+ const throwTheError = class { constructor() { throw theError; } };
+
+ // Test that the native side successfully captures the exception
+ let returnedError = test_exception.constructReturnException(throwTheError);
+ assert.strictEqual(returnedError, theError);
+
+ // Test that the native side passes the exception through
+ assert.throws(
+ () => { test_exception.constructAllowException(throwTheError); },
+ (err) => err === theError
+ );
+
+ // Test that the exception thrown above was marked as pending
+ // before it was handled on the JS side
+ const exception_pending = test_exception.wasPending();
+ assert.strictEqual(exception_pending, true,
+ 'Exception not pending as expected,' +
+ ` .wasPending() returned ${exception_pending}`);
+
+ // Test that the native side does not capture a non-existing exception
+ returnedError = test_exception.constructReturnException(common.mustCall());
+ assert.strictEqual(returnedError, undefined,
+ 'Returned error should be undefined when no exception is' +
+ ` thrown, but ${returnedError} was passed`);
+}
+
{
// Test that no exception appears that was not thrown by us
let caughtError;
@@ -66,3 +94,22 @@ const test_exception = (function() {
'Exception state did not remain clear as expected,' +
` .wasPending() returned ${exception_pending}`);
}
+
+{
+ // Test that no exception appears that was not thrown by us
+ let caughtError;
+ try {
+ test_exception.constructAllowException(common.mustCall());
+ } catch (anError) {
+ caughtError = anError;
+ }
+ assert.strictEqual(caughtError, undefined,
+ 'No exception originated on the native side, but' +
+ ` ${caughtError} was passed`);
+
+ // Test that the exception state remains clear when no exception is thrown
+ const exception_pending = test_exception.wasPending();
+ assert.strictEqual(exception_pending, false,
+ 'Exception state did not remain clear as expected,' +
+ ` .wasPending() returned ${exception_pending}`);
+}
diff --git a/test/js-native-api/test_exception/test_exception.c b/test/js-native-api/test_exception/test_exception.c
index 791f5219fc61fe..844f4475ac4d4c 100644
--- a/test/js-native-api/test_exception/test_exception.c
+++ b/test/js-native-api/test_exception/test_exception.c
@@ -22,6 +22,22 @@ static napi_value returnException(napi_env env, napi_callback_info info) {
return NULL;
}
+static napi_value constructReturnException(napi_env env, napi_callback_info info) {
+ size_t argc = 1;
+ napi_value args[1];
+ NODE_API_CALL(env, napi_get_cb_info(env, info, &argc, args, NULL, NULL));
+
+ napi_value result;
+ napi_status status = napi_new_instance(env, args[0], 0, 0, &result);
+ if (status == napi_pending_exception) {
+ napi_value ex;
+ NODE_API_CALL(env, napi_get_and_clear_last_exception(env, &ex));
+ return ex;
+ }
+
+ return NULL;
+}
+
static napi_value allowException(napi_env env, napi_callback_info info) {
size_t argc = 1;
napi_value args[1];
@@ -38,6 +54,19 @@ static napi_value allowException(napi_env env, napi_callback_info info) {
return NULL;
}
+static napi_value constructAllowException(napi_env env, napi_callback_info info) {
+ size_t argc = 1;
+ napi_value args[1];
+ NODE_API_CALL(env, napi_get_cb_info(env, info, &argc, args, NULL, NULL));
+
+ napi_value result;
+ napi_new_instance(env, args[0], 0, 0, &result);
+ // Ignore status and check napi_is_exception_pending() instead.
+
+ NODE_API_CALL(env, napi_is_exception_pending(env, &exceptionWasPending));
+ return NULL;
+}
+
static napi_value wasPending(napi_env env, napi_callback_info info) {
napi_value result;
NODE_API_CALL(env, napi_get_boolean(env, exceptionWasPending, &result));
@@ -64,6 +93,8 @@ napi_value Init(napi_env env, napi_value exports) {
napi_property_descriptor descriptors[] = {
DECLARE_NODE_API_PROPERTY("returnException", returnException),
DECLARE_NODE_API_PROPERTY("allowException", allowException),
+ DECLARE_NODE_API_PROPERTY("constructReturnException", constructReturnException),
+ DECLARE_NODE_API_PROPERTY("constructAllowException", constructAllowException),
DECLARE_NODE_API_PROPERTY("wasPending", wasPending),
DECLARE_NODE_API_PROPERTY("createExternal", createExternal),
};
From 03e75fda4cf1de0664c3110190886e684ece20bc Mon Sep 17 00:00:00 2001
From: Stephen Belanger
Date: Wed, 2 Jun 2021 22:22:15 -0700
Subject: [PATCH 017/118] async_hooks: switch between native and context hooks
correctly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
:facepalm: Might help if I remember to disable the _other_ promise
hook implementation when switching between them...
Fixes #38814
Fixes #38815
Refs #36394
PR-URL: https://github.com/nodejs/node/pull/38912
Reviewed-By: Vladimir de Turckheim
Reviewed-By: Gerhard Stöbich
Reviewed-By: Bryan English
---
lib/internal/async_hooks.js | 2 +
...ync-hooks-correctly-switch-promise-hook.js | 77 +++++++++++++++++++
2 files changed, 79 insertions(+)
create mode 100644 test/parallel/test-async-hooks-correctly-switch-promise-hook.js
diff --git a/lib/internal/async_hooks.js b/lib/internal/async_hooks.js
index eac2471ff79fb2..a6d258cf25757a 100644
--- a/lib/internal/async_hooks.js
+++ b/lib/internal/async_hooks.js
@@ -357,7 +357,9 @@ function updatePromiseHookMode() {
wantPromiseHook = true;
if (destroyHooksExist()) {
enablePromiseHook();
+ setPromiseHooks(undefined, undefined, undefined, undefined);
} else {
+ disablePromiseHook();
setPromiseHooks(
initHooksExist() ? promiseInitHook : undefined,
promiseBeforeHook,
diff --git a/test/parallel/test-async-hooks-correctly-switch-promise-hook.js b/test/parallel/test-async-hooks-correctly-switch-promise-hook.js
new file mode 100644
index 00000000000000..73127f1ebaf94c
--- /dev/null
+++ b/test/parallel/test-async-hooks-correctly-switch-promise-hook.js
@@ -0,0 +1,77 @@
+'use strict';
+require('../common');
+const assert = require('assert');
+const async_hooks = require('async_hooks');
+
+// Regression test for:
+// - https://github.com/nodejs/node/issues/38814
+// - https://github.com/nodejs/node/issues/38815
+
+const layers = new Map();
+
+// Only init to start context-based promise hook
+async_hooks.createHook({
+ init(asyncId, type) {
+ layers.set(asyncId, {
+ type,
+ init: true,
+ before: false,
+ after: false,
+ promiseResolve: false
+ });
+ },
+ before(asyncId) {
+ if (layers.has(asyncId)) {
+ layers.get(asyncId).before = true;
+ }
+ },
+ after(asyncId) {
+ if (layers.has(asyncId)) {
+ layers.get(asyncId).after = true;
+ }
+ },
+ promiseResolve(asyncId) {
+ if (layers.has(asyncId)) {
+ layers.get(asyncId).promiseResolve = true;
+ }
+ }
+}).enable();
+
+// With destroy, this should switch to native
+// and disable context - based promise hook
+async_hooks.createHook({
+ init() { },
+ destroy() { }
+}).enable();
+
+async function main() {
+ return Promise.resolve();
+}
+
+main();
+
+process.on('exit', () => {
+ assert.deepStrictEqual(Array.from(layers.values()), [
+ {
+ type: 'PROMISE',
+ init: true,
+ before: true,
+ after: true,
+ promiseResolve: true
+ },
+ {
+ type: 'PROMISE',
+ init: true,
+ before: false,
+ after: false,
+ promiseResolve: true
+ },
+ {
+ type: 'PROMISE',
+ init: true,
+ before: true,
+ after: true,
+ promiseResolve: true
+ },
+ ]);
+});
From 6e93c17bf51bad21441474f3a795760a2a2175d3 Mon Sep 17 00:00:00 2001
From: Shelley Vohr
Date: Wed, 2 Jun 2021 12:20:42 +0200
Subject: [PATCH 018/118] crypto: use EVP_get_cipherbynid directly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
PR-URL: https://github.com/nodejs/node/pull/38901
Reviewed-By: Richard Lau
Reviewed-By: James M Snell
Reviewed-By: Tobias Nießen
Reviewed-By: Anna Henningsen
---
src/crypto/crypto_cipher.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/crypto/crypto_cipher.cc b/src/crypto/crypto_cipher.cc
index 4629143d47e492..3ec212ee52b976 100644
--- a/src/crypto/crypto_cipher.cc
+++ b/src/crypto/crypto_cipher.cc
@@ -62,7 +62,7 @@ void GetCipherInfo(const FunctionCallbackInfo& args) {
cipher = EVP_get_cipherbyname(*name);
} else {
int nid = args[1].As()->Value();
- cipher = EVP_get_cipherbyname(OBJ_nid2sn(nid));
+ cipher = EVP_get_cipherbynid(nid);
}
if (cipher == nullptr)
From 6d5dc63ae406845933d90b6d6faaebc6cb63b1a6 Mon Sep 17 00:00:00 2001
From: Shelley Vohr
Date: Fri, 4 Jun 2021 11:11:54 +0200
Subject: [PATCH 019/118] crypto: fix label cast in
EVP_PKEY_CTX_set0_rsa_oaep_label
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
PR-URL: https://github.com/nodejs/node/pull/38926
Reviewed-By: Rich Trott
Reviewed-By: Anna Henningsen
Reviewed-By: Tobias Nießen
---
src/crypto/crypto_cipher.cc | 2 +-
src/crypto/crypto_rsa.cc | 5 ++++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/crypto/crypto_cipher.cc b/src/crypto/crypto_cipher.cc
index 3ec212ee52b976..198297d4edbd88 100644
--- a/src/crypto/crypto_cipher.cc
+++ b/src/crypto/crypto_cipher.cc
@@ -906,7 +906,7 @@ bool PublicKeyCipher::Cipher(
void* label = OPENSSL_memdup(oaep_label.data(), oaep_label.size());
CHECK_NOT_NULL(label);
if (0 >= EVP_PKEY_CTX_set0_rsa_oaep_label(ctx.get(),
- reinterpret_cast(label),
+ static_cast(label),
oaep_label.size())) {
OPENSSL_free(label);
return false;
diff --git a/src/crypto/crypto_rsa.cc b/src/crypto/crypto_rsa.cc
index 5fa91cce1a6ad2..5bbeb01ab58ac7 100644
--- a/src/crypto/crypto_rsa.cc
+++ b/src/crypto/crypto_rsa.cc
@@ -210,7 +210,10 @@ WebCryptoCipherStatus RSA_Cipher(
if (label_len > 0) {
void* label = OPENSSL_memdup(params.label.get(), label_len);
CHECK_NOT_NULL(label);
- if (EVP_PKEY_CTX_set0_rsa_oaep_label(ctx.get(), label, label_len) <= 0) {
+ if (EVP_PKEY_CTX_set0_rsa_oaep_label(
+ ctx.get(),
+ static_cast(label),
+ label_len) <= 0) {
OPENSSL_free(label);
return WebCryptoCipherStatus::FAILED;
}
From 06afb8df653d3a27b4580af369179f2f62f11ebe Mon Sep 17 00:00:00 2001
From: Shelley Vohr
Date: Tue, 1 Jun 2021 14:47:03 +0200
Subject: [PATCH 020/118] src: make InitializeOncePerProcess more flexible
PR-URL: https://github.com/nodejs/node/pull/38888
Reviewed-By: Joyee Cheung
Reviewed-By: Anna Henningsen
Reviewed-By: James M Snell
---
src/node.cc | 122 +++++++++++++++++++++++++------------------
src/node_internals.h | 13 +++++
2 files changed, 84 insertions(+), 51 deletions(-)
diff --git a/src/node.cc b/src/node.cc
index 6c601fb920a04c..a9afbd2682f785 100644
--- a/src/node.cc
+++ b/src/node.cc
@@ -233,7 +233,7 @@ int Environment::InitializeInspector(
return 0;
}
-#endif // HAVE_INSPECTOR && NODE_USE_V8_PLATFORM
+#endif // HAVE_INSPECTOR
#define ATOMIC_WAIT_EVENTS(V) \
V(kStartWait, "started") \
@@ -957,12 +957,26 @@ int InitializeNodeWithArgs(std::vector* argv,
}
InitializationResult InitializeOncePerProcess(int argc, char** argv) {
+ return InitializeOncePerProcess(argc, argv, kDefaultInitialization);
+}
+
+InitializationResult InitializeOncePerProcess(
+ int argc,
+ char** argv,
+ InitializationSettingsFlags flags) {
+ uint64_t init_flags = flags;
+ if (init_flags & kDefaultInitialization) {
+ init_flags = init_flags | kInitializeV8 | kInitOpenSSL | kRunPlatformInit;
+ }
+
// Initialized the enabled list for Debug() calls with system
// environment variables.
per_process::enabled_debug_list.Parse(nullptr);
atexit(ResetStdio);
- PlatformInit();
+
+ if (init_flags & kRunPlatformInit)
+ PlatformInit();
CHECK_GT(argc, 0);
@@ -1015,65 +1029,71 @@ InitializationResult InitializeOncePerProcess(int argc, char** argv) {
return result;
}
+ if (init_flags & kInitOpenSSL) {
#if HAVE_OPENSSL
- {
- std::string extra_ca_certs;
- if (credentials::SafeGetenv("NODE_EXTRA_CA_CERTS", &extra_ca_certs))
- crypto::UseExtraCaCerts(extra_ca_certs);
- }
- // In the case of FIPS builds we should make sure
- // the random source is properly initialized first.
-#if OPENSSL_VERSION_MAJOR >= 3
- // Call OPENSSL_init_crypto to initialize OPENSSL_INIT_LOAD_CONFIG to
- // avoid the default behavior where errors raised during the parsing of the
- // OpenSSL configuration file are not propagated and cannot be detected.
- //
- // If FIPS is configured the OpenSSL configuration file will have an .include
- // pointing to the fipsmodule.cnf file generated by the openssl fipsinstall
- // command. If the path to this file is incorrect no error will be reported.
- //
- // For Node.js this will mean that EntropySource will be called by V8 as part
- // of its initialization process, and EntropySource will in turn call
- // CheckEntropy. CheckEntropy will call RAND_status which will now always
- // return 0, leading to an endless loop and the node process will appear to
- // hang/freeze.
- std::string env_openssl_conf;
- credentials::SafeGetenv("OPENSSL_CONF", &env_openssl_conf);
-
- bool has_cli_conf = !per_process::cli_options->openssl_config.empty();
- if (has_cli_conf || !env_openssl_conf.empty()) {
- OPENSSL_INIT_SETTINGS* settings = OPENSSL_INIT_new();
- OPENSSL_INIT_set_config_file_flags(settings, CONF_MFLAGS_DEFAULT_SECTION);
- if (has_cli_conf) {
- const char* conf = per_process::cli_options->openssl_config.c_str();
- OPENSSL_INIT_set_config_filename(settings, conf);
+ {
+ std::string extra_ca_certs;
+ if (credentials::SafeGetenv("NODE_EXTRA_CA_CERTS", &extra_ca_certs))
+ crypto::UseExtraCaCerts(extra_ca_certs);
}
- OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, settings);
- OPENSSL_INIT_free(settings);
-
- if (ERR_peek_error() != 0) {
- result.exit_code = ERR_GET_REASON(ERR_peek_error());
- result.early_return = true;
- fprintf(stderr, "OpenSSL configuration error:\n");
- ERR_print_errors_fp(stderr);
- return result;
+ // In the case of FIPS builds we should make sure
+ // the random source is properly initialized first.
+#if OPENSSL_VERSION_MAJOR >= 3
+ // Call OPENSSL_init_crypto to initialize OPENSSL_INIT_LOAD_CONFIG to
+ // avoid the default behavior where errors raised during the parsing of the
+ // OpenSSL configuration file are not propagated and cannot be detected.
+ //
+ // If FIPS is configured the OpenSSL configuration file will have an
+ // .include pointing to the fipsmodule.cnf file generated by the openssl
+ // fipsinstall command. If the path to this file is incorrect no error
+ // will be reported.
+ //
+ // For Node.js this will mean that EntropySource will be called by V8 as
+ // part of its initialization process, and EntropySource will in turn call
+ // CheckEntropy. CheckEntropy will call RAND_status which will now always
+ // return 0, leading to an endless loop and the node process will appear to
+ // hang/freeze.
+ std::string env_openssl_conf;
+ credentials::SafeGetenv("OPENSSL_CONF", &env_openssl_conf);
+
+ bool has_cli_conf = !per_process::cli_options->openssl_config.empty();
+ if (has_cli_conf || !env_openssl_conf.empty()) {
+ OPENSSL_INIT_SETTINGS* settings = OPENSSL_INIT_new();
+ OPENSSL_INIT_set_config_file_flags(settings, CONF_MFLAGS_DEFAULT_SECTION);
+ if (has_cli_conf) {
+ const char* conf = per_process::cli_options->openssl_config.c_str();
+ OPENSSL_INIT_set_config_filename(settings, conf);
+ }
+ OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, settings);
+ OPENSSL_INIT_free(settings);
+
+ if (ERR_peek_error() != 0) {
+ result.exit_code = ERR_GET_REASON(ERR_peek_error());
+ result.early_return = true;
+ fprintf(stderr, "OpenSSL configuration error:\n");
+ ERR_print_errors_fp(stderr);
+ return result;
+ }
}
- }
#else
- if (FIPS_mode()) {
- OPENSSL_init();
- }
+ if (FIPS_mode()) {
+ OPENSSL_init();
+ }
#endif
- // V8 on Windows doesn't have a good source of entropy. Seed it from
- // OpenSSL's pool.
- V8::SetEntropySource(crypto::EntropySource);
+ // V8 on Windows doesn't have a good source of entropy. Seed it from
+ // OpenSSL's pool.
+ V8::SetEntropySource(crypto::EntropySource);
#endif // HAVE_OPENSSL
-
+}
per_process::v8_platform.Initialize(
static_cast(per_process::cli_options->v8_thread_pool_size));
- V8::Initialize();
+ if (init_flags & kInitializeV8) {
+ V8::Initialize();
+ }
+
performance::performance_v8_start = PERFORMANCE_NOW();
per_process::v8_initialized = true;
+
return result;
}
diff --git a/src/node_internals.h b/src/node_internals.h
index b75092d662dc97..31076551e70c46 100644
--- a/src/node_internals.h
+++ b/src/node_internals.h
@@ -316,7 +316,20 @@ struct InitializationResult {
std::vector exec_args;
bool early_return = false;
};
+
+enum InitializationSettingsFlags : uint64_t {
+ kDefaultInitialization = 1 << 0,
+ kInitializeV8 = 1 << 1,
+ kRunPlatformInit = 1 << 2,
+ kInitOpenSSL = 1 << 3
+};
+
+// TODO(codebytere): eventually document and expose to embedders.
InitializationResult InitializeOncePerProcess(int argc, char** argv);
+InitializationResult InitializeOncePerProcess(
+ int argc,
+ char** argv,
+ InitializationSettingsFlags flags);
void TearDownOncePerProcess();
void SetIsolateErrorHandlers(v8::Isolate* isolate, const IsolateSettings& s);
void SetIsolateMiscHandlers(v8::Isolate* isolate, const IsolateSettings& s);
From 651c58b41255aef338c66fd3a961727a00967199 Mon Sep 17 00:00:00 2001
From: bl-ue
Date: Tue, 1 Jun 2021 14:09:14 -0400
Subject: [PATCH 021/118] build: correct Xcode spelling in .gitignore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
PR-URL: https://github.com/nodejs/node/pull/38895
Reviewed-By: Michaël Zasso
Reviewed-By: Anna Henningsen
Reviewed-By: Colin Ihrig
Reviewed-By: Rich Trott
Reviewed-By: Darshan Sen
Reviewed-By: Zijian Liu
Reviewed-By: James M Snell
---
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index f2d8c226a698f1..b46679450bdbe6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -117,7 +117,7 @@ tools/*/*.i.tmp
/build
/coverage
-# === Rules for XCode artifacts ===
+# === Rules for Xcode artifacts ===
*.xcodeproj
*.xcworkspace
*.pbxproj
From e87cd4542b29098df25c2813382d8df559549c36 Mon Sep 17 00:00:00 2001
From: Qingyu Deng
Date: Fri, 4 Jun 2021 17:49:04 +0800
Subject: [PATCH 022/118] child_process: refactor to use `validateBoolean`
PR-URL: https://github.com/nodejs/node/pull/38927
Reviewed-By: Antoine du Hamel
Reviewed-By: Darshan Sen
Reviewed-By: Khaidi Chu
Reviewed-By: Zijian Liu
Reviewed-By: James M Snell
---
lib/child_process.js | 21 ++++++++-------------
1 file changed, 8 insertions(+), 13 deletions(-)
diff --git a/lib/child_process.js b/lib/child_process.js
index e3691639a9fddd..73268b05a85740 100644
--- a/lib/child_process.js
+++ b/lib/child_process.js
@@ -75,6 +75,7 @@ const { getValidatedPath } = require('internal/fs/utils');
const {
isInt32,
validateAbortSignal,
+ validateBoolean,
validateObject,
validateString,
} = require('internal/validators');
@@ -459,10 +460,8 @@ function normalizeSpawnArguments(file, args, options) {
}
// Validate detached, if present.
- if (options.detached != null &&
- typeof options.detached !== 'boolean') {
- throw new ERR_INVALID_ARG_TYPE('options.detached',
- 'boolean', options.detached);
+ if (options.detached != null) {
+ validateBoolean(options.detached, 'options.detached');
}
// Validate the uid, if present.
@@ -489,19 +488,15 @@ function normalizeSpawnArguments(file, args, options) {
}
// Validate windowsHide, if present.
- if (options.windowsHide != null &&
- typeof options.windowsHide !== 'boolean') {
- throw new ERR_INVALID_ARG_TYPE('options.windowsHide',
- 'boolean', options.windowsHide);
+ if (options.windowsHide != null) {
+ validateBoolean(options.windowsHide, 'options.windowsHide');
}
// Validate windowsVerbatimArguments, if present.
let { windowsVerbatimArguments } = options;
- if (windowsVerbatimArguments != null &&
- typeof windowsVerbatimArguments !== 'boolean') {
- throw new ERR_INVALID_ARG_TYPE('options.windowsVerbatimArguments',
- 'boolean',
- windowsVerbatimArguments);
+ if (windowsVerbatimArguments != null) {
+ validateBoolean(windowsVerbatimArguments,
+ 'options.windowsVerbatimArguments');
}
if (options.shell) {
From f40725f2a1d536e1f2b59fcadb5c5085bbe74917 Mon Sep 17 00:00:00 2001
From: Voltrex <62040526+VoltrexMaster@users.noreply.github.com>
Date: Fri, 4 Jun 2021 23:11:25 +0430
Subject: [PATCH 023/118] vm: use missing validator
The `vm` lib module's `isContext()` function should use a validator.
PR-URL: https://github.com/nodejs/node/pull/38935
Reviewed-By: Gus Caplan
Reviewed-By: Zijian Liu
Reviewed-By: Anna Henningsen
Reviewed-By: Darshan Sen
Reviewed-By: Luigi Pinca
Reviewed-By: James M Snell
---
lib/vm.js | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/lib/vm.js b/lib/vm.js
index 0b643110ae9465..d8e3d6586036f3 100644
--- a/lib/vm.js
+++ b/lib/vm.js
@@ -206,9 +206,8 @@ function getContextOptions(options) {
}
function isContext(object) {
- if (typeof object !== 'object' || object === null) {
- throw new ERR_INVALID_ARG_TYPE('object', 'Object', object);
- }
+ validateObject(object, 'object', { allowArray: true });
+
return _isContext(object);
}
From 0f65e414428fac6558673a7dbe937f8f3114cdbe Mon Sep 17 00:00:00 2001
From: Rich Trott
Date: Sat, 5 Jun 2021 18:57:30 -0700
Subject: [PATCH 024/118] debugger: reduce scope of eslint disable comment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Current code masks setInterval and setTimeout with promisified versions.
This can be confusing to read and causes lint errors. Replace masking
with use of pSetInterval and pSetTimeout instead.
Move disabling of lint rule from entire file to the one remaining line
(after the above changes) that still needs it.
PR-URL: https://github.com/nodejs/node/pull/38946
Reviewed-By: Antoine du Hamel
Reviewed-By: Michaël Zasso
Reviewed-By: Anna Henningsen
Reviewed-By: James M Snell
Reviewed-By: Luigi Pinca
---
lib/internal/inspector/_inspect.js | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/lib/internal/inspector/_inspect.js b/lib/internal/inspector/_inspect.js
index 427469b4a6f116..6f8e389e4212ea 100644
--- a/lib/internal/inspector/_inspect.js
+++ b/lib/internal/inspector/_inspect.js
@@ -20,9 +20,6 @@
* IN THE SOFTWARE.
*/
-// TODO(aduh95): remove restricted syntax errors
-/* eslint-disable no-restricted-syntax */
-
'use strict';
const {
@@ -53,8 +50,8 @@ const { EventEmitter } = require('events');
const net = require('net');
const util = require('util');
const {
- setInterval,
- setTimeout,
+ setInterval: pSetInterval,
+ setTimeout: pSetTimeout,
} = require('timers/promises');
const {
AbortController,
@@ -85,13 +82,13 @@ async function portIsFree(host, port, timeout = 9999) {
const ac = new AbortController();
const { signal } = ac;
- setTimeout(timeout).then(() => ac.abort());
+ pSetTimeout(timeout).then(() => ac.abort());
- const asyncIterator = setInterval(retryDelay);
+ const asyncIterator = pSetInterval(retryDelay);
while (true) {
await asyncIterator.next();
if (signal.aborted) {
- throw new StartupError(
+ throw new StartupError( // eslint-disable-line no-restricted-syntax
`Timeout (${timeout}) waiting for ${host}:${port} to be free`);
}
const error = await new Promise((resolve) => {
@@ -251,7 +248,7 @@ class NodeInspector {
return;
} catch (error) {
debuglog('connect failed', error);
- await setTimeout(1000);
+ await pSetTimeout(1000);
}
}
this.stdout.write(' failed to connect, please retry\n');
From 9ba5518f08226f91ce2c959c4f5c05233698053c Mon Sep 17 00:00:00 2001
From: Daniel Bevenius
Date: Tue, 25 May 2021 05:33:22 +0200
Subject: [PATCH 025/118] src: skip test_fatal/test_threads for Debug builds
Currently test/node-api/test_fatal/test_threads.js fails for a Debug
build with the following error:
1: 0x101e3f8 node::DumpBacktrace(_IO_FILE*) [/node/out/Debug/node]
2: 0x11c31ed [/node/out/Debug/node]
3: 0x11c320d [/node/out/Debug/node]
4: 0x2ba4448 V8_Fatal(char const*, int, char const*, ...) [/node/out/Debug/node]
5: 0x2ba4473 [/node/out/Debug/node]
6: 0x139e049 v8::internal::Isolate::Current() [/node/out/Debug/node]
7: 0x11025ee node::OnFatalError(char const*, char const*) [/node/out/Debug/node]
8: 0x1102564 node::FatalError(char const*, char const*) [/node/out/Debug/node]
9: 0x10add1d napi_open_callback_scope [/node/out/Debug/node]
10: 0x7f05664211dc [/node/test/node-api/test_fatal/build/Debug/test_fatal.node]
11: 0x7f056608e4e2 [/usr/lib64/libpthread.so.0]
12: 0x7f0565fbd6c3 clone [/usr/lib64/libc.so.6]
node:assert:412
throw err;
^
AssertionError [ERR_ASSERTION]: The expression evaluated to a falsy value:
assert.ok(p.status === 134 || p.signal === 'SIGABRT')
at Object. (/node/test/node-api/test_fatal/test_threads.js:21:8)
at Module._compile (node:internal/modules/cjs/loader:1109:14)
at Object.Module._extensions..js (node:internal/modules/cjs/loader:1138:10)
at Module.load (node:internal/modules/cjs/loader:989:32)
at Function.Module._load (node:internal/modules/cjs/loader:829:14)
at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:79:12)
at node:internal/main/run_main_module:17:47 {
generatedMessage: true,
code: 'ERR_ASSERTION',
actual: false,
expected: true,
operator: '=='
}
This is caused by a call to Isolate::GetCurrent() when the calling
thread has not initialized V8. We are working suggestion to add a method
to V8 which allows a check/get without any checks but in the mean time
this change should allow debug builds to pass the test suit.
PR-URL: https://github.com/nodejs/node/pull/38805
Refs: https://chromium-review.googlesource.com/c/v8/v8/+/2910630
Reviewed-By: Michael Dawson
Reviewed-By: James M Snell
---
test/node-api/test_fatal/test_threads.js | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/test/node-api/test_fatal/test_threads.js b/test/node-api/test_fatal/test_threads.js
index fd56f60cbe832f..6f6dc772d18252 100644
--- a/test/node-api/test_fatal/test_threads.js
+++ b/test/node-api/test_fatal/test_threads.js
@@ -4,6 +4,10 @@ const assert = require('assert');
const child_process = require('child_process');
const test_fatal = require(`./build/${common.buildType}/test_fatal`);
+if (common.buildType === 'Debug')
+ common.skip('as this will currently fail with a Debug check ' +
+ 'in v8::Isolate::GetCurrent()');
+
// Test in a child process because the test code will trigger a fatal error
// that crashes the process.
if (process.argv[2] === 'child') {
From 1799ea36f0dcf0e6eb1faa641e0969c319757eb7 Mon Sep 17 00:00:00 2001
From: Shelley Vohr
Date: Fri, 4 Jun 2021 10:37:03 +0200
Subject: [PATCH 026/118] crypto: use compatible version of EVP_CIPHER_name
PR-URL: https://github.com/nodejs/node/pull/38925
Reviewed-By: Rich Trott
Reviewed-By: Richard Lau
Reviewed-By: Anna Henningsen
Reviewed-By: Franziska Hinkelmann
---
src/crypto/crypto_cipher.cc | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/crypto/crypto_cipher.cc b/src/crypto/crypto_cipher.cc
index 198297d4edbd88..6abfc2cce7da79 100644
--- a/src/crypto/crypto_cipher.cc
+++ b/src/crypto/crypto_cipher.cc
@@ -145,10 +145,14 @@ void GetCipherInfo(const FunctionCallbackInfo& args) {
return;
}
+ // OBJ_nid2sn(EVP_CIPHER_nid(cipher)) is used here instead of
+ // EVP_CIPHER_name(cipher) for compatibility with BoringSSL.
if (info->Set(
env->context(),
env->name_string(),
- OneByteString(env->isolate(), EVP_CIPHER_name(cipher))).IsNothing()) {
+ OneByteString(
+ env->isolate(),
+ OBJ_nid2sn(EVP_CIPHER_nid(cipher)))).IsNothing()) {
return;
}
From 08b2a4a13809d00e7736fe07c2eeda9d777cde69 Mon Sep 17 00:00:00 2001
From: Daniel Bevenius
Date: Mon, 31 May 2021 06:08:01 +0200
Subject: [PATCH 027/118] src,test: raise error for --enable-fips when no FIPS
This commit moves the check for FIPS from the crypto module
initialization to process startup.
The motivation for this is that when OpenSSL is not FIPS enabled and the
command line options --enable-fips, or --force-fips are used, there will
only be an error raised if the crypto module is used. This can be
surprising and we have gotten feedback that users assumed that there
would be an error if these options were specified and FIPS is not
available.
PR-URL: https://github.com/nodejs/node/pull/38859
Reviewed-By: Michael Dawson
Reviewed-By: James M Snell
Reviewed-By: Anna Henningsen
Reviewed-By: Richard Lau
---
src/crypto/crypto_util.cc | 40 +++++++++++++++----------------
src/crypto/crypto_util.h | 2 ++
src/node.cc | 14 ++++++++---
test/parallel/test-crypto-fips.js | 23 ++++++++++++++++--
4 files changed, 53 insertions(+), 26 deletions(-)
diff --git a/src/crypto/crypto_util.cc b/src/crypto/crypto_util.cc
index bc4efe5f597263..13c40dcb757661 100644
--- a/src/crypto/crypto_util.cc
+++ b/src/crypto/crypto_util.cc
@@ -14,11 +14,9 @@
#include "math.h"
-#ifdef OPENSSL_FIPS
#if OPENSSL_VERSION_MAJOR >= 3
#include "openssl/provider.h"
#endif
-#endif
#include
@@ -107,6 +105,25 @@ int NoPasswordCallback(char* buf, int size, int rwflag, void* u) {
return 0;
}
+bool ProcessFipsOptions() {
+ /* Override FIPS settings in configuration file, if needed. */
+ if (per_process::cli_options->enable_fips_crypto ||
+ per_process::cli_options->force_fips_crypto) {
+#if OPENSSL_VERSION_MAJOR >= 3
+ OSSL_PROVIDER* fips_provider = OSSL_PROVIDER_load(nullptr, "fips");
+ if (fips_provider == nullptr)
+ return false;
+ OSSL_PROVIDER_unload(fips_provider);
+
+ return EVP_default_properties_enable_fips(nullptr, 1) &&
+ EVP_default_properties_is_fips_enabled(nullptr);
+#else
+ return FIPS_mode() == 0 && FIPS_mode_set(1);
+#endif
+ }
+ return true;
+}
+
void InitCryptoOnce() {
#ifndef OPENSSL_IS_BORINGSSL
OPENSSL_INIT_SETTINGS* settings = OPENSSL_INIT_new();
@@ -143,25 +160,6 @@ void InitCryptoOnce() {
}
#endif
- /* Override FIPS settings in cnf file, if needed. */
- unsigned long err = 0; // NOLINT(runtime/int)
- if (per_process::cli_options->enable_fips_crypto ||
- per_process::cli_options->force_fips_crypto) {
-#if OPENSSL_VERSION_MAJOR >= 3
- if (0 == EVP_default_properties_is_fips_enabled(nullptr) &&
- !EVP_default_properties_enable_fips(nullptr, 1)) {
-#else
- if (0 == FIPS_mode() && !FIPS_mode_set(1)) {
-#endif
- err = ERR_get_error();
- }
- }
- if (0 != err) {
- auto* isolate = Isolate::GetCurrent();
- auto* env = Environment::GetCurrent(isolate);
- return ThrowCryptoError(env, err);
- }
-
// Turn off compression. Saves memory and protects against CRIME attacks.
// No-op with OPENSSL_NO_COMP builds of OpenSSL.
sk_SSL_COMP_zero(SSL_COMP_get_compression_methods());
diff --git a/src/crypto/crypto_util.h b/src/crypto/crypto_util.h
index 94bcb100cca0e2..ac95612a0b1a85 100644
--- a/src/crypto/crypto_util.h
+++ b/src/crypto/crypto_util.h
@@ -86,6 +86,8 @@ using DsaSigPointer = DeleteFnPtr;
// callback has been made.
extern int VerifyCallback(int preverify_ok, X509_STORE_CTX* ctx);
+bool ProcessFipsOptions();
+
void InitCryptoOnce();
void InitCrypto(v8::Local target);
diff --git a/src/node.cc b/src/node.cc
index a9afbd2682f785..3ca2a05d8b8b96 100644
--- a/src/node.cc
+++ b/src/node.cc
@@ -1080,9 +1080,17 @@ InitializationResult InitializeOncePerProcess(
OPENSSL_init();
}
#endif
- // V8 on Windows doesn't have a good source of entropy. Seed it from
- // OpenSSL's pool.
- V8::SetEntropySource(crypto::EntropySource);
+ if (!crypto::ProcessFipsOptions()) {
+ result.exit_code = ERR_GET_REASON(ERR_peek_error());
+ result.early_return = true;
+ fprintf(stderr, "OpenSSL error when trying to enable FIPS:\n");
+ ERR_print_errors_fp(stderr);
+ return result;
+ }
+
+ // V8 on Windows doesn't have a good source of entropy. Seed it from
+ // OpenSSL's pool.
+ V8::SetEntropySource(crypto::EntropySource);
#endif // HAVE_OPENSSL
}
per_process::v8_platform.Initialize(
diff --git a/test/parallel/test-crypto-fips.js b/test/parallel/test-crypto-fips.js
index b6e70b62be68b9..ba8a1ba653ec55 100644
--- a/test/parallel/test-crypto-fips.js
+++ b/test/parallel/test-crypto-fips.js
@@ -17,6 +17,7 @@ const FIPS_ERROR_STRING2 =
'Error [ERR_CRYPTO_FIPS_FORCED]: Cannot set FIPS mode, it was forced with ' +
'--force-fips at startup.';
const FIPS_UNSUPPORTED_ERROR_STRING = 'fips mode not supported';
+const FIPS_ENABLE_ERROR_STRING = 'OpenSSL error when trying to enable FIPS:';
const CNF_FIPS_ON = fixtures.path('openssl_fips_enabled.cnf');
const CNF_FIPS_OFF = fixtures.path('openssl_fips_disabled.cnf');
@@ -49,8 +50,10 @@ function testHelper(stream, args, expectedOutput, cmd, env) {
// In the case of expected errors just look for a substring.
assert.ok(response.includes(expectedOutput));
} else {
- // Normal path where we expect either FIPS enabled or disabled.
- assert.strictEqual(Number(response), expectedOutput);
+ const getFipsValue = Number(response);
+ if (!Number.isNaN(getFipsValue))
+ // Normal path where we expect either FIPS enabled or disabled.
+ assert.strictEqual(getFipsValue, expectedOutput);
}
childOk(child);
}
@@ -58,6 +61,22 @@ function testHelper(stream, args, expectedOutput, cmd, env) {
responseHandler(child[stream], expectedOutput);
}
+// --enable-fips should raise an error if OpenSSL is not FIPS enabled.
+testHelper(
+ testFipsCrypto() ? 'stdout' : 'stderr',
+ ['--enable-fips'],
+ testFipsCrypto() ? FIPS_ENABLED : FIPS_ENABLE_ERROR_STRING,
+ 'process.versions',
+ process.env);
+
+// --force-fips should raise an error if OpenSSL is not FIPS enabled.
+testHelper(
+ testFipsCrypto() ? 'stdout' : 'stderr',
+ ['--force-fips'],
+ testFipsCrypto() ? FIPS_ENABLED : FIPS_ENABLE_ERROR_STRING,
+ 'process.versions',
+ process.env);
+
// By default FIPS should be off in both FIPS and non-FIPS builds.
testHelper(
'stdout',
From 5b5e07a2ccad8e13d4a2f7669517b831581f696e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C3=ABl=20Zasso?=
Date: Sun, 6 Jun 2021 12:16:54 +0200
Subject: [PATCH 028/118] meta: update label-pr-config
- Rename "ES Modules" label to "esm" (This already happened on GitHub
a while ago).
- Add missing "fast-track" to deps/npm (It was lost when the pull
request adding it landed).
- Rename "intl" to "i18n-api" (There is no intl label).
- Rename "url-whatwg" to "whatwg-url".
- Rename "V8 Engine" to "v8 engine".
- Rename "n-api" to "node-api".
- Add "python" to .py files.
- Add "gyp" to tools/gyp.
- Add "icu" to tools/icu.
- Add "tools" and "v8 engine" to tools/v8_gypfiles.
- Add "release" to doc/changelogs.
PR-URL: https://github.com/nodejs/node/pull/38950
Reviewed-By: Antoine du Hamel
Reviewed-By: Rich Trott
Reviewed-By: James M Snell
Reviewed-By: Richard Lau
---
.github/label-pr-config.yml | 50 ++++++++++++++++++-------------------
1 file changed, 25 insertions(+), 25 deletions(-)
diff --git a/.github/label-pr-config.yml b/.github/label-pr-config.yml
index ca23908800ed2b..6e1bccd478991f 100644
--- a/.github/label-pr-config.yml
+++ b/.github/label-pr-config.yml
@@ -11,7 +11,7 @@ subSystemLabels:
/^src\/udp_/: c++, dgram
/^src\/(?:fs_|node_file|node_stat_watcher)/: c++, fs
/^src\/node_http_parser/: c++, http_parser
- /^src\/node_i18n/: c++, intl
+ /^src\/node_i18n/: c++, i18n-api
/^src\/uv\./: c++, libuv
/^src\/(?:connect(?:ion)?|pipe|tcp)_/: c++, net
/^src\/node_os/: c++, os
@@ -19,14 +19,14 @@ subSystemLabels:
/^src\/timer_/: c++, timers
/^src\/(?:CNNICHashWhitelist|node_root_certs|tls_)/: c++, tls
/^src\/tty_/: c++, tty
- /^src\/node_url/: c++, url-whatwg
+ /^src\/node_url/: c++, whatwg-url
/^src\/node_util/: c++, util
- /^src\/(?:node_v8|v8abbr)/: c++, V8 Engine
+ /^src\/(?:node_v8|v8abbr)/: c++, v8 engine
/^src\/node_contextify/: c++, vm
/^src\/.*win32.*/: c++, windows
/^src\/node_zlib/: c++, zlib
/^src\/tracing/: c++, tracing
- /^src\/node_api/: c++, n-api
+ /^src\/node_api/: c++, node-api
/^src\/node_http2/: c++, http2
/^src\/node_report/: c++, report
/^src\/node_wasi/: c++, wasi
@@ -35,7 +35,7 @@ subSystemLabels:
/^src\/node_bob*/: c++, quic, dont-land-on-v14.x, dont-land-on-v12.x
# don't label python files as c++
- /^src\/.+\.py$/: lib / src, needs-ci
+ /^src\/.+\.py$/: python, needs-ci
# properly label changes to v8 inspector integration-related files
/^src\/inspector_/: c++, inspector, needs-ci
@@ -50,13 +50,13 @@ subSystemLabels:
/^\w+\.md$/: doc
# different variants of *Makefile and build files
/^(tools\/)?(Makefile|BSDmakefile|create_android_makefiles|\.travis\.yml)$/: build, needs-ci
- /^tools\/(install\.py|genv8constants\.py|getnodeversion\.py|js2c\.py|utils\.py|configure\.d\/.*)$/: build, needs-ci
+ /^tools\/(install\.py|genv8constants\.py|getnodeversion\.py|js2c\.py|utils\.py|configure\.d\/.*)$/: build, python, needs-ci
/^vcbuild\.bat$/: build, windows, needs-ci
/^(android-)?configure|node\.gyp|common\.gypi$/: build, needs-ci
# more specific tools
- /^tools\/gyp/: tools, build, needs-ci, dont-land-on-v14.x, dont-land-on-v12.x
+ /^tools\/gyp/: tools, build, gyp, needs-ci, dont-land-on-v14.x, dont-land-on-v12.x
/^tools\/doc\//: tools, doc
- /^tools\/icu\//: tools, intl, needs-ci
+ /^tools\/icu\//: tools, i18n-api, icu, needs-ci
/^tools\/(?:osx-pkg\.pmdoc|pkgsrc)\//: tools, macos, install
/^tools\/(?:(?:mac)?osx-)/: tools, macos
/^tools\/test-npm/: tools, test, npm
@@ -64,9 +64,10 @@ subSystemLabels:
/^tools\/(?:certdata|mkssldef|mk-ca-bundle)/: tools, openssl, tls
/^tools\/msvs\//: tools, windows, install, needs-ci
/^tools\/[^/]+\.bat$/: tools, windows, needs-ci
- /^tools\/make-v8/: tools, V8 Engine, needs-ci
- /^tools\/(code_cache|snapshot|v8_gypfiles)/: needs-ci,
- /^tools\/build-addons.js/: needs-ci,
+ /^tools\/make-v8/: tools, v8 engine, needs-ci
+ /^tools\/v8_gypfiles/: tools, v8 engine, needs-ci
+ /^tools\/(code_cache|snapshot)/: needs-ci
+ /^tools\/build-addons.js/: needs-ci
# all other tool changes should be marked as such
/^tools\//: tools
/^\.eslint|\.remark|\.editorconfig/: tools
@@ -75,10 +76,10 @@ subSystemLabels:
# libuv needs an explicit mapping, as the ordinary /deps/ mapping below would
# end up as libuv changes labeled with "uv" (which is a non-existing label)
/^deps\/uv\//: libuv
- /^deps\/v8\/tools\/gen-postmortem-metadata\.py/: V8 Engine, post-mortem
- /^deps\/v8\//: V8 Engine
+ /^deps\/v8\/tools\/gen-postmortem-metadata\.py/: v8 engine, python, post-mortem
+ /^deps\/v8\//: v8 engine
/^deps\/uvwasi\//: wasi
- /^deps\/npm\//: npm, dont-land-on-v14.x, dont-land-on-v12.x
+ /^deps\/npm\//: npm, fast-track, dont-land-on-v14.x, dont-land-on-v12.x
/^deps\/nghttp2\/nghttp2\.gyp/: build, http2
/^deps\/nghttp2\//: http2
/^deps\/ngtcp2\//: quic, dont-land-on-v14.x, dont-land-on-v12.x
@@ -97,8 +98,8 @@ subSystemLabels:
/^lib\/\w+\/streams$/: stream
/^lib\/.*http2/: http2
/^lib\/worker_threads.js$/: worker
- /^lib\/internal\/url\.js$/: url-whatwg
- /^lib\/internal\/modules\/esm/: ES Modules
+ /^lib\/internal\/url\.js$/: whatwg-url
+ /^lib\/internal\/modules\/esm/: esm
/^lib\/internal\/quic\/*/: quic, dont-land-on-v14.x, dont-land-on-v12.x
# All other lib/ files map directly
@@ -115,12 +116,12 @@ exlusiveLabels:
/^test\/pseudo-tty\//: test, tty
/^test\/inspector\//: test, inspector
/^test\/cctest\/test_inspector/: test, inspector
- /^test\/cctest\/test_url/: test, url-whatwg
- /^test\/addons-napi\//: test, n-api
+ /^test\/cctest\/test_url/: test, whatwg-url
+ /^test\/addons-napi\//: test, node-api
/^test\/async-hooks\//: test, async_hooks
/^test\/report\//: test, report
- /^test\/fixtures\/es-module/: test, ES Modules
- /^test\/es-module\//: test, ES Modules
+ /^test\/fixtures\/es-module/: test, esm
+ /^test\/es-module\//: test, esm
/^test\//: test
@@ -128,11 +129,9 @@ exlusiveLabels:
/^doc\/api\/webcrypto.md$/: doc, crypto
# specific map for modules.md as it should be labeled 'module' not 'modules'
/^doc\/api\/modules.md$/: doc, module
- # specific map for esm.md as it should be labeled 'ES Modules' not 'esm'
- /^doc\/api\/esm.md$/: doc, ES Modules
- # n-api is treated separately since it is not a JS core module but is still
+ # node-api is treated separately since it is not a JS core module but is still
# considered a subsystem of sorts
- /^doc\/api\/n-api.md$/: doc, n-api
+ /^doc\/api\/n-api.md$/: doc, node-api
# quic
/^doc\/api\/quic.md$/: doc, quic, dont-land-on-v14.x, dont-land-on-v12.x
# add worker label to PRs that affect doc/api/worker_threads.md
@@ -141,12 +140,13 @@ exlusiveLabels:
/^doc\/api\/(\w+)\.md$/: doc, $1
# add deprecations label to PRs that affect doc/api/deprecations.md
/^doc\/api\/deprecations.md$/: doc, deprecations
+ /^doc\/changelogs\//: release
/^doc\//: doc
# more specific benchmarks
/^benchmark\/buffers\//: benchmark, buffer
- /^benchmark\/(?:arrays|es)\//: benchmark, V8 Engine
+ /^benchmark\/(?:arrays|es)\//: benchmark, v8 engine
/^benchmark\/_http/: benchmark, http
/^benchmark\/(?:misc|fixtures)\//: benchmark
/^benchmark\/streams\//: benchmark, stream
From ec3e5b4c15c6712fdb2174f4ced5a6afbbee3e9d Mon Sep 17 00:00:00 2001
From: Michael Dawson
Date: Tue, 1 Jun 2021 21:20:55 -0400
Subject: [PATCH 029/118] node-api: avoid SecondPassCallback crash
PR https://github.com/nodejs/node/pull/38000 added
indirection so that we could stop finalization in
cases where it had been scheduled in a second
pass callback but we were doing it in advance in
environment teardown.
Unforunately we missed that the code which tries
to clear the second pass parameter checked if
the pointer to the parameter (_secondPassParameter)
was nullptr and that when the second pass callback
was scheduled we set _secondPassParameter to nullptr
in order to avoid it being deleted outside of the second
pass callback. The net result was that we
would not clear the _secondPassParameter contents
and failed to avoid the Finalization in the second pass
callback.
This PR adds an additional boolean for deciding if
the secondPassParameter should be deleted outside
of the second pass callback instead of setting
secondPassParameter to nullptr thus avoiding the
conflict between the 2 ways it was being used.
See the discussion starting at:
https://github.com/nodejs/node/pull/38273#issuecomment-852403751
for how this was discovered on OSX while trying to
upgrade to a new V8 version.
Signed-off-by: Michael Dawson
PR-URL: https://github.com/nodejs/node/pull/38899
Reviewed-By: Chengzhong Wu
Reviewed-By: James M Snell
---
src/js_native_api_v8.cc | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/js_native_api_v8.cc b/src/js_native_api_v8.cc
index 95f8212d870a72..d972ee43c8861e 100644
--- a/src/js_native_api_v8.cc
+++ b/src/js_native_api_v8.cc
@@ -321,7 +321,8 @@ class Reference : public RefBase {
Reference(napi_env env, v8::Local value, Args&&... args)
: RefBase(env, std::forward(args)...),
_persistent(env->isolate, value),
- _secondPassParameter(new SecondPassCallParameterRef(this)) {
+ _secondPassParameter(new SecondPassCallParameterRef(this)),
+ _secondPassScheduled(false) {
if (RefCount() == 0) {
SetWeak();
}
@@ -348,7 +349,7 @@ class Reference : public RefBase {
// If the second pass callback is scheduled, it will delete the
// parameter passed to it, otherwise it will never be scheduled
// and we need to delete it here.
- if (_secondPassParameter != nullptr) {
+ if (!_secondPassScheduled) {
delete _secondPassParameter;
}
}
@@ -445,8 +446,7 @@ class Reference : public RefBase {
reference->_persistent.Reset();
// Mark the parameter not delete-able until the second pass callback is
// invoked.
- reference->_secondPassParameter = nullptr;
-
+ reference->_secondPassScheduled = true;
data.SetSecondPassCallback(SecondPassCallback);
}
@@ -468,12 +468,14 @@ class Reference : public RefBase {
// the reference itself has already been deleted so nothing to do
return;
}
+ reference->_secondPassParameter = nullptr;
reference->Finalize();
}
bool env_teardown_finalize_started_ = false;
v8impl::Persistent _persistent;
SecondPassCallParameterRef* _secondPassParameter;
+ bool _secondPassScheduled;
};
enum UnwrapAction {
From 336571fbdd2a48976acfbacb0b686f56223b1596 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C3=ABl=20Zasso?=
Date: Sun, 6 Jun 2021 11:51:23 +0200
Subject: [PATCH 030/118] Revert "http: make HEAD method to work with
keep-alive"
This reverts commit 7afa5336aed999a62e4943e6000a239585b2e2ea.
The change breaks clients like cURL.
Fixes: https://github.com/nodejs/node/issues/38922
PR-URL: https://github.com/nodejs/node/pull/38949
Reviewed-By: Colin Ihrig
Reviewed-By: Matteo Collina
Reviewed-By: James M Snell
Reviewed-By: Mary Marchini
Reviewed-By: Michael Dawson
Reviewed-By: Robert Nagy
Reviewed-By: Jiawen Geng
---
lib/_http_outgoing.js | 3 +-
test/parallel/test-http-reuse-socket.js | 51 -------------------------
2 files changed, 1 insertion(+), 53 deletions(-)
delete mode 100644 test/parallel/test-http-reuse-socket.js
diff --git a/lib/_http_outgoing.js b/lib/_http_outgoing.js
index 4d3b58cc84d00c..c6f68d4329c7da 100644
--- a/lib/_http_outgoing.js
+++ b/lib/_http_outgoing.js
@@ -459,8 +459,7 @@ function _storeHeader(firstLine, headers) {
}
if (!state.contLen && !state.te) {
- if (!this._hasBody && (this.statusCode === 204 ||
- this.statusCode === 304)) {
+ if (!this._hasBody) {
// Make sure we don't end the 0\r\n\r\n at the end of the message.
this.chunkedEncoding = false;
} else if (!this.useChunkedEncodingByDefault) {
diff --git a/test/parallel/test-http-reuse-socket.js b/test/parallel/test-http-reuse-socket.js
deleted file mode 100644
index f5cd002fdbf519..00000000000000
--- a/test/parallel/test-http-reuse-socket.js
+++ /dev/null
@@ -1,51 +0,0 @@
-'use strict';
-const common = require('../common');
-const http = require('http');
-const assert = require('assert');
-const Countdown = require('../common/countdown');
-
-// The HEAD:204, GET:200 is the most pathological test case.
-// GETs following a 204 response with a content-encoding header failed.
-// Responses without bodies and without content-length or encoding caused
-// the socket to be closed.
-const codes = [204, 200, 200, 304, 200];
-const methods = ['HEAD', 'HEAD', 'GET', 'HEAD', 'GET'];
-
-const sockets = [];
-const agent = new http.Agent();
-agent.maxSockets = 1;
-
-const countdown = new Countdown(codes.length, () => server.close());
-
-const server = http.createServer(common.mustCall((req, res) => {
- const code = codes.shift();
- assert.strictEqual(typeof code, 'number');
- assert.ok(code > 0);
- res.writeHead(code, {});
- res.end();
-}, codes.length));
-
-function nextRequest() {
- const request = http.request({
- port: server.address().port,
- path: '/',
- agent: agent,
- method: methods.shift()
- }, common.mustCall((response) => {
- response.on('end', common.mustCall(() => {
- if (countdown.dec()) {
- nextRequest();
- }
- assert.strictEqual(sockets.length, 1);
- }));
- response.resume();
- }));
- request.on('socket', common.mustCall((socket) => {
- if (!sockets.includes(socket)) {
- sockets.push(socket);
- }
- }));
- request.end();
-}
-
-server.listen(0, common.mustCall(nextRequest));
From 70af1467451ab72fd267c53a60160ec9eb6cf95c Mon Sep 17 00:00:00 2001
From: npm-robot
Date: Thu, 3 Jun 2021 20:17:35 +0000
Subject: [PATCH 031/118] deps: upgrade npm to 7.16.0
PR-URL: https://github.com/nodejs/node/pull/38920
Reviewed-By: Ruy Adorno
Reviewed-By: Myles Borins
---
deps/npm/.npmignore | 4 +
deps/npm/CHANGELOG.md | 64 ++
.../docs/content/commands/npm-run-script.md | 7 +-
deps/npm/docs/content/using-npm/config.md | 2 +
deps/npm/docs/output/commands/npm-ls.html | 4 +-
.../docs/output/commands/npm-run-script.html | 6 +-
deps/npm/docs/output/commands/npm.html | 4 +-
deps/npm/docs/output/using-npm/config.html | 1 +
deps/npm/lib/cli.js | 2 +-
deps/npm/lib/utils/config/definitions.js | 2 +
deps/npm/lib/utils/error-handler.js | 4 +-
deps/npm/lib/utils/update-notifier.js | 36 +-
deps/npm/man/man1/npm-access.1 | 2 +-
deps/npm/man/man1/npm-adduser.1 | 2 +-
deps/npm/man/man1/npm-audit.1 | 2 +-
deps/npm/man/man1/npm-bin.1 | 2 +-
deps/npm/man/man1/npm-bugs.1 | 2 +-
deps/npm/man/man1/npm-cache.1 | 2 +-
deps/npm/man/man1/npm-ci.1 | 2 +-
deps/npm/man/man1/npm-completion.1 | 2 +-
deps/npm/man/man1/npm-config.1 | 2 +-
deps/npm/man/man1/npm-dedupe.1 | 2 +-
deps/npm/man/man1/npm-deprecate.1 | 2 +-
deps/npm/man/man1/npm-diff.1 | 2 +-
deps/npm/man/man1/npm-dist-tag.1 | 2 +-
deps/npm/man/man1/npm-docs.1 | 2 +-
deps/npm/man/man1/npm-doctor.1 | 2 +-
deps/npm/man/man1/npm-edit.1 | 2 +-
deps/npm/man/man1/npm-exec.1 | 2 +-
deps/npm/man/man1/npm-explain.1 | 2 +-
deps/npm/man/man1/npm-explore.1 | 2 +-
deps/npm/man/man1/npm-find-dupes.1 | 2 +-
deps/npm/man/man1/npm-fund.1 | 2 +-
deps/npm/man/man1/npm-help-search.1 | 2 +-
deps/npm/man/man1/npm-help.1 | 2 +-
deps/npm/man/man1/npm-hook.1 | 2 +-
deps/npm/man/man1/npm-init.1 | 2 +-
deps/npm/man/man1/npm-install-ci-test.1 | 2 +-
deps/npm/man/man1/npm-install-test.1 | 2 +-
deps/npm/man/man1/npm-install.1 | 2 +-
deps/npm/man/man1/npm-link.1 | 2 +-
deps/npm/man/man1/npm-logout.1 | 2 +-
deps/npm/man/man1/npm-ls.1 | 4 +-
deps/npm/man/man1/npm-org.1 | 2 +-
deps/npm/man/man1/npm-outdated.1 | 2 +-
deps/npm/man/man1/npm-owner.1 | 2 +-
deps/npm/man/man1/npm-pack.1 | 2 +-
deps/npm/man/man1/npm-ping.1 | 2 +-
deps/npm/man/man1/npm-prefix.1 | 2 +-
deps/npm/man/man1/npm-profile.1 | 2 +-
deps/npm/man/man1/npm-prune.1 | 2 +-
deps/npm/man/man1/npm-publish.1 | 2 +-
deps/npm/man/man1/npm-rebuild.1 | 2 +-
deps/npm/man/man1/npm-repo.1 | 2 +-
deps/npm/man/man1/npm-restart.1 | 2 +-
deps/npm/man/man1/npm-root.1 | 2 +-
deps/npm/man/man1/npm-run-script.1 | 8 +-
deps/npm/man/man1/npm-search.1 | 2 +-
deps/npm/man/man1/npm-set-script.1 | 2 +-
deps/npm/man/man1/npm-shrinkwrap.1 | 2 +-
deps/npm/man/man1/npm-star.1 | 2 +-
deps/npm/man/man1/npm-stars.1 | 2 +-
deps/npm/man/man1/npm-start.1 | 2 +-
deps/npm/man/man1/npm-stop.1 | 2 +-
deps/npm/man/man1/npm-team.1 | 2 +-
deps/npm/man/man1/npm-test.1 | 2 +-
deps/npm/man/man1/npm-token.1 | 2 +-
deps/npm/man/man1/npm-uninstall.1 | 2 +-
deps/npm/man/man1/npm-unpublish.1 | 2 +-
deps/npm/man/man1/npm-unstar.1 | 2 +-
deps/npm/man/man1/npm-update.1 | 2 +-
deps/npm/man/man1/npm-version.1 | 2 +-
deps/npm/man/man1/npm-view.1 | 2 +-
deps/npm/man/man1/npm-whoami.1 | 2 +-
deps/npm/man/man1/npm.1 | 4 +-
deps/npm/man/man1/npx.1 | 2 +-
deps/npm/man/man5/folders.5 | 2 +-
deps/npm/man/man5/install.5 | 2 +-
deps/npm/man/man5/npm-shrinkwrap-json.5 | 2 +-
deps/npm/man/man5/npmrc.5 | 2 +-
deps/npm/man/man5/package-json.5 | 2 +-
deps/npm/man/man5/package-lock-json.5 | 2 +-
deps/npm/man/man7/config.7 | 4 +-
deps/npm/man/man7/developers.7 | 2 +-
deps/npm/man/man7/orgs.7 | 2 +-
deps/npm/man/man7/registry.7 | 2 +-
deps/npm/man/man7/removal.7 | 2 +-
deps/npm/man/man7/scope.7 | 2 +-
deps/npm/man/man7/scripts.7 | 2 +-
deps/npm/man/man7/workspaces.7 | 2 +-
.../@npmcli/arborist/package.json | 4 +-
.../iconv-lite/.idea/codeStyles/Project.xml | 47 ++
.../.idea/codeStyles/codeStyleConfig.xml | 5 +
.../iconv-lite/.idea/iconv-lite.iml | 12 +
.../inspectionProfiles/Project_Default.xml | 6 +
.../node_modules/iconv-lite/.idea/modules.xml | 8 +
.../npm/node_modules/iconv-lite/.idea/vcs.xml | 6 +
deps/npm/node_modules/iconv-lite/Changelog.md | 4 +
.../iconv-lite/encodings/dbcs-data.js | 14 +-
deps/npm/node_modules/iconv-lite/package.json | 2 +-
.../node_modules/libnpmaccess/package.json | 4 +-
deps/npm/node_modules/libnpmhook/CHANGELOG.md | 110 ---
deps/npm/node_modules/libnpmhook/package.json | 4 +-
deps/npm/node_modules/libnpmorg/CHANGELOG.md | 33 -
deps/npm/node_modules/libnpmorg/package.json | 4 +-
.../node_modules/libnpmpublish/CHANGELOG.md | 91 ---
.../node_modules/libnpmpublish/package.json | 4 +-
.../node_modules/libnpmsearch/CHANGELOG.md | 57 --
.../node_modules/libnpmsearch/package.json | 4 +-
deps/npm/node_modules/libnpmteam/CHANGELOG.md | 40 --
deps/npm/node_modules/libnpmteam/package.json | 4 +-
.../make-fetch-happen/CHANGELOG.md | 654 ------------------
.../node_modules/make-fetch-happen/README.md | 31 +-
.../node_modules/make-fetch-happen/cache.js | 260 -------
.../node_modules/make-fetch-happen/index.js | 457 ------------
.../make-fetch-happen/{ => lib}/agent.js | 37 +-
.../make-fetch-happen/lib/cache/entry.js | 432 ++++++++++++
.../make-fetch-happen/lib/cache/errors.js | 10 +
.../make-fetch-happen/lib/cache/index.js | 46 ++
.../make-fetch-happen/lib/cache/key.js | 17 +
.../make-fetch-happen/lib/cache/policy.js | 161 +++++
.../make-fetch-happen/lib/fetch.js | 100 +++
.../make-fetch-happen/lib/index.js | 40 ++
.../make-fetch-happen/lib/options.js | 45 ++
.../make-fetch-happen/lib/remote.js | 101 +++
.../make-fetch-happen/package.json | 34 +-
.../utils/configure-options.js | 32 -
.../utils/initialize-cache.js | 26 -
.../utils/is-header-conditional.js | 17 -
.../utils/iterable-to-object.js | 9 -
.../make-fetch-happen/utils/make-policy.js | 19 -
.../node_modules/make-fetch-happen/warning.js | 24 -
deps/npm/node_modules/mime-db/HISTORY.md | 7 +
deps/npm/node_modules/mime-db/db.json | 58 +-
deps/npm/node_modules/mime-db/package.json | 14 +-
deps/npm/node_modules/mime-types/HISTORY.md | 8 +
deps/npm/node_modules/mime-types/package.json | 14 +-
deps/npm/node_modules/negotiator/HISTORY.md | 103 +++
deps/npm/node_modules/negotiator/LICENSE | 24 +
deps/npm/node_modules/negotiator/README.md | 203 ++++++
deps/npm/node_modules/negotiator/index.js | 124 ++++
.../node_modules/negotiator/lib/charset.js | 169 +++++
.../node_modules/negotiator/lib/encoding.js | 184 +++++
.../node_modules/negotiator/lib/language.js | 179 +++++
.../node_modules/negotiator/lib/mediaType.js | 294 ++++++++
deps/npm/node_modules/negotiator/package.json | 42 ++
.../node_modules/npm-package-arg/CHANGELOG.md | 52 --
deps/npm/node_modules/npm-package-arg/npa.js | 144 ++--
.../node_modules/npm-package-arg/package.json | 12 +-
.../npm/node_modules/npm-profile/CHANGELOG.md | 62 --
.../npm/node_modules/npm-profile/package.json | 4 +-
.../node_modules/npm-registry-fetch/README.md | 7 +-
.../npm-registry-fetch/check-response.js | 83 +--
.../node_modules/npm-registry-fetch/index.js | 3 +-
.../npm-registry-fetch/package.json | 21 +-
deps/npm/node_modules/pacote/package.json | 4 +-
deps/npm/node_modules/path-parse/.travis.yml | 9 -
deps/npm/node_modules/path-parse/index.js | 50 +-
deps/npm/node_modules/path-parse/package.json | 2 +-
deps/npm/node_modules/path-parse/test.js | 77 ---
.../node_modules}/form-data/License | 0
.../node_modules}/form-data/README.md | 0
.../node_modules}/form-data/README.md.bak | 0
.../node_modules}/form-data/lib/browser.js | 0
.../node_modules}/form-data/lib/form_data.js | 0
.../node_modules}/form-data/lib/populate.js | 0
.../node_modules}/form-data/package.json | 0
.../node_modules}/form-data/yarn.lock | 0
.../node_modules/spdx-license-ids/README.md | 2 +-
.../node_modules/spdx-license-ids/index.json | 12 +
.../spdx-license-ids/package.json | 2 +-
deps/npm/package.json | 24 +-
.../lib/utils/config/describe-all.js.test.cjs | 2 +
deps/npm/test/lib/cli.js | 1 +
deps/npm/test/lib/utils/update-notifier.js | 61 +-
175 files changed, 2906 insertions(+), 2470 deletions(-)
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/codeStyles/Project.xml
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/codeStyles/codeStyleConfig.xml
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/iconv-lite.iml
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/inspectionProfiles/Project_Default.xml
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/modules.xml
create mode 100644 deps/npm/node_modules/iconv-lite/.idea/vcs.xml
delete mode 100644 deps/npm/node_modules/libnpmhook/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/libnpmorg/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/libnpmpublish/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/libnpmsearch/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/libnpmteam/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/make-fetch-happen/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/make-fetch-happen/cache.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/index.js
rename deps/npm/node_modules/make-fetch-happen/{ => lib}/agent.js (88%)
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/cache/entry.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/cache/errors.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/cache/index.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/cache/key.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/cache/policy.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/fetch.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/index.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/options.js
create mode 100644 deps/npm/node_modules/make-fetch-happen/lib/remote.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/utils/configure-options.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/utils/initialize-cache.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/utils/is-header-conditional.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/utils/iterable-to-object.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/utils/make-policy.js
delete mode 100644 deps/npm/node_modules/make-fetch-happen/warning.js
create mode 100644 deps/npm/node_modules/negotiator/HISTORY.md
create mode 100644 deps/npm/node_modules/negotiator/LICENSE
create mode 100644 deps/npm/node_modules/negotiator/README.md
create mode 100644 deps/npm/node_modules/negotiator/index.js
create mode 100644 deps/npm/node_modules/negotiator/lib/charset.js
create mode 100644 deps/npm/node_modules/negotiator/lib/encoding.js
create mode 100644 deps/npm/node_modules/negotiator/lib/language.js
create mode 100644 deps/npm/node_modules/negotiator/lib/mediaType.js
create mode 100644 deps/npm/node_modules/negotiator/package.json
delete mode 100644 deps/npm/node_modules/npm-package-arg/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/npm-profile/CHANGELOG.md
delete mode 100644 deps/npm/node_modules/path-parse/.travis.yml
delete mode 100644 deps/npm/node_modules/path-parse/test.js
rename deps/npm/node_modules/{ => request/node_modules}/form-data/License (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/README.md (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/README.md.bak (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/lib/browser.js (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/lib/form_data.js (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/lib/populate.js (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/package.json (100%)
rename deps/npm/node_modules/{ => request/node_modules}/form-data/yarn.lock (100%)
diff --git a/deps/npm/.npmignore b/deps/npm/.npmignore
index 9d02b99f91b39a..ae91e6482791fa 100644
--- a/deps/npm/.npmignore
+++ b/deps/npm/.npmignore
@@ -27,6 +27,10 @@ docs/nav.yml
docs/config.json
docs/dockhand.js
docs/template.html
+docs/package.json
+docs/node_modules
+# docs source files are required by `npm help-search` do not exclude those
+!docs/content/
# don't ignore .npmignore files
# these are used in some tests.
diff --git a/deps/npm/CHANGELOG.md b/deps/npm/CHANGELOG.md
index c86373bcde2b84..027731cbe2cf73 100644
--- a/deps/npm/CHANGELOG.md
+++ b/deps/npm/CHANGELOG.md
@@ -1,3 +1,67 @@
+## v7.16.0 (2021-06-03)
+
+## FEATURES
+
+* [`e92b5f2ba`](https://github.com/npm/cli/commit/e92b5f2ba07746ae07646566f3dc73c9e004a2fc)
+ `npm-registry-fetch@11.0.0`
+ * feat: improved logging of cache status
+
+## BUG FIXES
+
+* [`e864bd3ce`](https://github.com/npm/cli/commit/e864bd3ce8e8467e0f8ebb499dc2daf06143bc33)
+ [#3345](https://github.com/npm/cli/issues/3345)
+ fix(update-notifier): do not update notify when installing npm@spec
+ ([@isaacs](https://github.com/isaacs))
+* [`aafe23572`](https://github.com/npm/cli/commit/aafe2357279230e333d3342752a28fce6b9cd152)
+ [#3348](https://github.com/npm/cli/issues/3348)
+ fix(update-notifier): parallelize check for updates
+ ([@isaacs](https://github.com/isaacs))
+
+## DOCUMENTATION
+
+* [`bc9c57dda`](https://github.com/npm/cli/commit/bc9c57dda7cf3abcdee17550205daf1a82e90438)
+ [#3353](https://github.com/npm/cli/issues/3353)
+ fix(docs): remove documentation for '--scripts-prepend-node-path' as it was removed in npm@7
+ ([@gimli01](https://github.com/gimli01))
+* [`ca2822110`](https://github.com/npm/cli/commit/ca28221103aa0e9ccba7043ac515a541b625c53a)
+ [#3360](https://github.com/npm/cli/issues/3360)
+ fix(docs): link foreground-scripts w/ loglevel
+ ([@wraithgar](https://github.com/wraithgar))
+* [`fb630b5a9`](https://github.com/npm/cli/commit/fb630b5a9af86c71602803297634ec291eeedee0)
+ [#3342](https://github.com/npm/cli/issues/3342)
+ chore(docs): manage docs as a workspace
+ ([@ruyadorno](https://github.com/ruyadorno))
+
+## DEPENDENCIES
+
+* [`54de5c6a4`](https://github.com/npm/cli/commit/54de5c6a4cd593bbbe364132f3f7348586441b31)
+ `npm-package-arg@8.1.4`:
+ * fix: trim whitespace from fetchSpec
+ * fix: handle file: when root directory begins with a special character
+* [`e92b5f2ba`](https://github.com/npm/cli/commit/e92b5f2ba07746ae07646566f3dc73c9e004a2fc)
+ `make-fetch-happen@9.0.1`
+ * breaking: complete refactor of caching. drops warning headers,
+ prevents cache indexes from growing for every request, correctly
+ handles varied requests to the same url, and now caches redirects.
+ * fix: support url-encoded proxy authorization
+ * fix: do not lazy-load proxy agents or agentkeepalive. fixes the
+ intermittent failures to update npm on slower connections.
+ `npm-registry-fetch@11.0.0`
+ * breaking: drop handling of deprecated warning headers
+ * docs: fix header type for npm-command
+ * docs: update registry param
+ * feat: improved logging of cache status
+* [`23c50a45f`](https://github.com/npm/cli/commit/23c50a45f59ea3ed4c36f35df15e54adc5603034)
+ `make-fetch-happen@9.0.2`:
+ * fix: work around negotiator's lazy loading
+
+## AUTOMATION
+
+* [`c4ef78b08`](https://github.com/npm/cli/commit/c4ef78b08e6859fc191cabbe58c8d88c070e0612)
+ [#3344](https://github.com/npm/cli/issues/3344)
+ fix(automation): update incorrect variable name in create-cli-deps-pr workflow
+ ([@gimli01](https://github.com/gimli01))
+
## v7.15.1 (2021-05-31)
### BUG FIXES
diff --git a/deps/npm/docs/content/commands/npm-run-script.md b/deps/npm/docs/content/commands/npm-run-script.md
index 1d11a74faa2448..5e3828c40717dd 100644
--- a/deps/npm/docs/content/commands/npm-run-script.md
+++ b/deps/npm/docs/content/commands/npm-run-script.md
@@ -70,11 +70,7 @@ can use the `INIT_CWD` environment variable, which holds the full path you
were in when you ran `npm run`.
`npm run` sets the `NODE` environment variable to the `node` executable
-with which `npm` is executed. Also, if the `--scripts-prepend-node-path` is
-passed, the directory within which `node` resides is added to the `PATH`.
-If `--scripts-prepend-node-path=auto` is passed (which has been the default
-in `npm` v3), this is only performed when that `node` executable is not
-found in the `PATH`.
+with which `npm` is executed.
If you try to run a script without having a `node_modules` directory and it
fails, you will be given a warning to run `npm install`, just in case you've
@@ -138,7 +134,6 @@ npm test -w a -w b
This last command will run `test` in both `./packages/a` and `./packages/b`
packages.
-
### Configuration
diff --git a/deps/npm/docs/content/using-npm/config.md b/deps/npm/docs/content/using-npm/config.md
index 25b4d424e82ff4..44b79a801f15ec 100644
--- a/deps/npm/docs/content/using-npm/config.md
+++ b/deps/npm/docs/content/using-npm/config.md
@@ -776,6 +776,8 @@ What level of logs to report. On failure, *all* logs are written to
Any logs of a higher level than the setting are shown. The default is
"notice".
+See also the `foreground-scripts` config.
+
#### `logs-max`
* Default: 10
diff --git a/deps/npm/docs/output/commands/npm-ls.html b/deps/npm/docs/output/commands/npm-ls.html
index f87185539dd6f7..1b33d79cc1a807 100644
--- a/deps/npm/docs/output/commands/npm-ls.html
+++ b/deps/npm/docs/output/commands/npm-ls.html
@@ -159,7 +159,7 @@ Description
the results to only the paths to the packages named. Note that nested
packages will also show the paths to the specified packages. For
example, running npm ls promzard
in npm’s source tree will show:
-npm@7.15.1 /path/to/npm
+npm@7.16.0 /path/to/npm
└─┬ init-package-json@0.0.4
└── promzard@0.1.5
@@ -337,4 +337,4 @@ See Also
-