From b801b228fed6b6d5c5f1c9875f8e30ae6430513c Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 21:59:14 +0100 Subject: [PATCH 01/15] bootstrap: support module_wrap binding in snapshot --- src/module_wrap.cc | 20 ++++++++++++++++++++ src/module_wrap.h | 2 ++ src/node_external_reference.h | 1 + 3 files changed, 23 insertions(+) diff --git a/src/module_wrap.cc b/src/module_wrap.cc index 73ce4aa42035a1..d6c766244fd416 100644 --- a/src/module_wrap.cc +++ b/src/module_wrap.cc @@ -4,6 +4,7 @@ #include "memory_tracker-inl.h" #include "node_contextify.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" #include "node_process-inl.h" #include "node_url.h" @@ -811,8 +812,27 @@ void ModuleWrap::Initialize(Local target, #undef V } +void ModuleWrap::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(New); + + registry->Register(Link); + registry->Register(Instantiate); + registry->Register(Evaluate); + registry->Register(SetSyntheticExport); + registry->Register(CreateCachedData); + registry->Register(GetNamespace); + registry->Register(GetStatus); + registry->Register(GetError); + registry->Register(GetStaticDependencySpecifiers); + + registry->Register(SetImportModuleDynamicallyCallback); + registry->Register(SetInitializeImportMetaObjectCallback); +} } // namespace loader } // namespace node NODE_BINDING_CONTEXT_AWARE_INTERNAL(module_wrap, node::loader::ModuleWrap::Initialize) +NODE_BINDING_EXTERNAL_REFERENCE( + module_wrap, node::loader::ModuleWrap::RegisterExternalReferences) diff --git a/src/module_wrap.h b/src/module_wrap.h index 58b233d036515c..c609ba5509dcd0 100644 --- a/src/module_wrap.h +++ b/src/module_wrap.h @@ -11,6 +11,7 @@ namespace node { class Environment; +class ExternalReferenceRegistry; namespace contextify { class ContextifyContext; @@ -44,6 +45,7 @@ class ModuleWrap : public BaseObject { v8::Local unused, v8::Local context, void* priv); + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); static void HostInitializeImportMetaObjectCallback( v8::Local context, v8::Local module, diff --git a/src/node_external_reference.h b/src/node_external_reference.h index bf4b49670de310..c3ab57c0bb0f98 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -74,6 +74,7 @@ class ExternalReferenceRegistry { V(heap_utils) \ V(messaging) \ V(mksnapshot) \ + V(module_wrap) \ V(options) \ V(os) \ V(performance) \ From 43db8f850bc91c6714cb0fcd7ffeb25d0bc09775 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:03:24 +0100 Subject: [PATCH 02/15] bootstrap: include event_target into the built-in snapshot Since the module has to be loaded during bootstrap anyway. --- lib/internal/bootstrap/browser.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/internal/bootstrap/browser.js b/lib/internal/bootstrap/browser.js index 705ddc49dd8ade..b94ac9891a399c 100644 --- a/lib/internal/bootstrap/browser.js +++ b/lib/internal/bootstrap/browser.js @@ -42,9 +42,11 @@ defineOperation(globalThis, 'setTimeout', timers.setTimeout); exposeLazyInterfaces(globalThis, 'internal/abort_controller', [ 'AbortController', 'AbortSignal', ]); -exposeLazyInterfaces(globalThis, 'internal/event_target', [ - 'EventTarget', 'Event', -]); +const { + EventTarget, Event, +} = require('internal/event_target'); +exposeInterface(globalThis, 'Event', Event); +exposeInterface(globalThis, 'EventTarget', EventTarget); exposeLazyInterfaces(globalThis, 'internal/worker/io', [ 'MessageChannel', 'MessagePort', 'MessageEvent', ]); From b9509a1fa0b86c622043980c8e67a06b2e1b9f2c Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:24:28 +0100 Subject: [PATCH 03/15] modules: move modules/cjs/helpers.js to modules/helpers.js The helpers are actually shared by the two loaders, so move them under modules/ directly. --- lib/internal/main/eval_string.js | 2 +- lib/internal/modules/cjs/loader.js | 2 +- lib/internal/modules/esm/translators.js | 2 +- lib/internal/modules/{cjs => }/helpers.js | 0 lib/internal/source_map/source_map_cache.js | 2 +- lib/internal/util/inspector.js | 2 +- lib/repl.js | 2 +- test/parallel/test-bootstrap-modules.js | 2 +- test/parallel/test-util-inspect.js | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename lib/internal/modules/{cjs => }/helpers.js (100%) diff --git a/lib/internal/main/eval_string.js b/lib/internal/main/eval_string.js index e92e088b8e0ceb..e8ecf9f1dae9cf 100644 --- a/lib/internal/main/eval_string.js +++ b/lib/internal/main/eval_string.js @@ -14,7 +14,7 @@ const { markBootstrapComplete } = require('internal/process/pre_execution'); const { evalModule, evalScript } = require('internal/process/execution'); -const { addBuiltinLibsToObject } = require('internal/modules/cjs/helpers'); +const { addBuiltinLibsToObject } = require('internal/modules/helpers'); const { getOptionValue } = require('internal/options'); diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 36efb48fc574cb..466c160506e669 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -103,7 +103,7 @@ const { makeRequireFunction, normalizeReferrerURL, stripBOM, -} = require('internal/modules/cjs/helpers'); +} = require('internal/modules/helpers'); const { getOptionValue } = require('internal/options'); const preserveSymlinks = getOptionValue('--preserve-symlinks'); const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main'); diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 38fa41e47603dd..2007b8d5ab063d 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -30,7 +30,7 @@ const { hasEsmSyntax, loadBuiltinModule, stripBOM, -} = require('internal/modules/cjs/helpers'); +} = require('internal/modules/helpers'); const { Module: CJSModule, cjsParseCache diff --git a/lib/internal/modules/cjs/helpers.js b/lib/internal/modules/helpers.js similarity index 100% rename from lib/internal/modules/cjs/helpers.js rename to lib/internal/modules/helpers.js diff --git a/lib/internal/source_map/source_map_cache.js b/lib/internal/source_map/source_map_cache.js index 1eb5c69bd3ff93..8699c4e9509681 100644 --- a/lib/internal/source_map/source_map_cache.js +++ b/lib/internal/source_map/source_map_cache.js @@ -27,7 +27,7 @@ const { getOptionValue } = require('internal/options'); const { IterableWeakMap } = require('internal/util/iterable_weak_map'); const { normalizeReferrerURL, -} = require('internal/modules/cjs/helpers'); +} = require('internal/modules/helpers'); const { validateBoolean } = require('internal/validators'); const { setMaybeCacheGeneratedSourceMap } = internalBinding('errors'); diff --git a/lib/internal/util/inspector.js b/lib/internal/util/inspector.js index c7f18ffdb61a33..3a589423b2d280 100644 --- a/lib/internal/util/inspector.js +++ b/lib/internal/util/inspector.js @@ -66,7 +66,7 @@ function installConsoleExtensions(commandLineApi) { if (commandLineApi.require) { return; } const { tryGetCwd } = require('internal/process/execution'); const CJSModule = require('internal/modules/cjs/loader').Module; - const { makeRequireFunction } = require('internal/modules/cjs/helpers'); + const { makeRequireFunction } = require('internal/modules/helpers'); const consoleAPIModule = new CJSModule(''); const cwd = tryGetCwd(); consoleAPIModule.paths = ArrayPrototypeConcat( diff --git a/lib/repl.js b/lib/repl.js index 0780b5a54743c6..60e454e5374960 100644 --- a/lib/repl.js +++ b/lib/repl.js @@ -103,7 +103,7 @@ const { BuiltinModule } = require('internal/bootstrap/loaders'); const { makeRequireFunction, addBuiltinLibsToObject -} = require('internal/modules/cjs/helpers'); +} = require('internal/modules/helpers'); const { isIdentifierStart, isIdentifierChar diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index d3111513424182..bf2ca2e935c70f 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -53,7 +53,7 @@ const expectedModules = new Set([ 'NativeModule internal/fs/utils', 'NativeModule internal/idna', 'NativeModule internal/linkedlist', - 'NativeModule internal/modules/cjs/helpers', + 'NativeModule internal/modules/helpers', 'NativeModule internal/modules/cjs/loader', 'NativeModule internal/modules/esm/assert', 'NativeModule internal/modules/esm/formats', diff --git a/test/parallel/test-util-inspect.js b/test/parallel/test-util-inspect.js index 57f8dd064005c6..2e0d726d3dcecb 100644 --- a/test/parallel/test-util-inspect.js +++ b/test/parallel/test-util-inspect.js @@ -2832,7 +2832,7 @@ assert.strictEqual( ' at Function.Module._load (node:internal/modules/cjs/loader:621:3)', // This file is not an actual Node.js core file. ' at Module.require [as weird/name] (node:internal/aaaaa/loader:735:19)', - ' at require (node:internal/modules/cjs/helpers:14:16)', + ' at require (node:internal/modules/helpers:14:16)', ' at Array.forEach ()', ` at ${process.cwd()}/test/parallel/test-util-inspect.js:2760:12`, ` at Object. (${process.cwd()}/node_modules/hyper_module/folder/file.js:2753:10)`, From 9c65244a45164ec451da294f85c4ded225e52f94 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 00:02:51 +0100 Subject: [PATCH 04/15] lib: add getLazy() method to internal/util This patch adds a getLazy() method to facilitate initialize-once lazy loading in the internals. --- lib/internal/util.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/internal/util.js b/lib/internal/util.js index fd4a6e3eca3421..ecb88334ef4a82 100644 --- a/lib/internal/util.js +++ b/lib/internal/util.js @@ -654,7 +654,25 @@ function isArrayBufferDetached(value) { return false; } +/** + * @template T return value of loader + * @param {()=>T} loader + * @returns {()=>T} + */ +function getLazy(loader) { + let value; + let initialized = false; + return function() { + if (initialized === false) { + value = loader(); + initialized = true; + } + return value; + }; +} + module.exports = { + getLazy, assertCrypto, cachedResult, convertToValidSignal, From 30e18ee3d6b23d3c4f229385910e35aef183f621 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:32:02 +0100 Subject: [PATCH 05/15] lib: lazy-load deps in source_map_cache.js So that the file can be snapshotted. --- lib/internal/source_map/source_map_cache.js | 20 ++++++++++++-------- test/parallel/test-bootstrap-modules.js | 1 - 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/internal/source_map/source_map_cache.js b/lib/internal/source_map/source_map_cache.js index 8699c4e9509681..2faf4489059b1e 100644 --- a/lib/internal/source_map/source_map_cache.js +++ b/lib/internal/source_map/source_map_cache.js @@ -24,17 +24,19 @@ let debug = require('internal/util/debuglog').debuglog('source_map', (fn) => { debug = fn; }); const { getOptionValue } = require('internal/options'); -const { IterableWeakMap } = require('internal/util/iterable_weak_map'); -const { - normalizeReferrerURL, -} = require('internal/modules/helpers'); + const { validateBoolean } = require('internal/validators'); const { setMaybeCacheGeneratedSourceMap } = internalBinding('errors'); +const { getLazy } = require('internal/util'); // Since the CJS module cache is mutable, which leads to memory leaks when // modules are deleted, we use a WeakMap so that the source map cache will // be purged automatically: -const cjsSourceMapCache = new IterableWeakMap(); +const getCjsSourceMapCache = getLazy(() => { + const { IterableWeakMap } = require('internal/util/iterable_weak_map'); + return new IterableWeakMap(); +}); + // The esm cache is not mutable, so we can use a Map without memory concerns: const esmSourceMapCache = new SafeMap(); // The generated sources is not mutable, so we can use a Map without memory concerns: @@ -44,6 +46,7 @@ const kSourceMappingURLMagicComment = /\/[*/]#\s+sourceMappingURL=(?[^\s]+)/g; const { fileURLToPath, pathToFileURL, URL } = require('internal/url'); + let SourceMap; let sourceMapsEnabled; @@ -114,6 +117,7 @@ function maybeCacheSourceMap(filename, content, cjsModuleInstance, isGeneratedSo const sourceMapsEnabled = getSourceMapsEnabled(); if (!(process.env.NODE_V8_COVERAGE || sourceMapsEnabled)) return; try { + const { normalizeReferrerURL } = require('internal/modules/helpers'); filename = normalizeReferrerURL(filename); } catch (err) { // This is most likely an invalid filename in sourceURL of [eval]-wrapper. @@ -137,7 +141,7 @@ function maybeCacheSourceMap(filename, content, cjsModuleInstance, isGeneratedSo const data = dataFromUrl(filename, sourceMapURL); const url = data ? null : sourceMapURL; if (cjsModuleInstance) { - cjsSourceMapCache.set(cjsModuleInstance, { + getCjsSourceMapCache().set(cjsModuleInstance, { filename, lineLengths: lineLengths(content), data, @@ -291,7 +295,7 @@ function sourceMapCacheToObject() { } function appendCJSCache(obj) { - for (const value of cjsSourceMapCache) { + for (const value of getCjsSourceMapCache()) { obj[ObjectGetValueSafe(value, 'filename')] = { lineLengths: ObjectGetValueSafe(value, 'lineLengths'), data: ObjectGetValueSafe(value, 'data'), @@ -309,7 +313,7 @@ function findSourceMap(sourceURL) { } let sourceMap = esmSourceMapCache.get(sourceURL) ?? generatedSourceMapCache.get(sourceURL); if (sourceMap === undefined) { - for (const value of cjsSourceMapCache) { + for (const value of getCjsSourceMapCache()) { const filename = ObjectGetValueSafe(value, 'filename'); const cachedSourceURL = ObjectGetValueSafe(value, 'sourceURL'); if (sourceURL === filename || sourceURL === cachedSourceURL) { diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index bf2ca2e935c70f..064a208eb7b56f 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -87,7 +87,6 @@ const expectedModules = new Set([ 'NativeModule internal/util', 'NativeModule internal/util/debuglog', 'NativeModule internal/util/inspect', - 'NativeModule internal/util/iterable_weak_map', 'NativeModule internal/util/types', 'NativeModule internal/v8/startup_snapshot', 'NativeModule internal/validators', From e263a6102e64f080bd2564289270c7f083756fa2 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:35:42 +0100 Subject: [PATCH 06/15] lib: lazy-load deps in modules/run_main.js So that the file can be snapshotted --- lib/internal/modules/run_main.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 738c945bc21c21..c948eaf4ae4437 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -4,8 +4,7 @@ const { ObjectCreate, StringPrototypeEndsWith, } = primordials; -const CJSLoader = require('internal/modules/cjs/loader'); -const { Module, toRealPath, readPackageScope } = CJSLoader; + const { getOptionValue } = require('internal/options'); const path = require('path'); @@ -13,6 +12,7 @@ function resolveMainPath(main) { // Note extension resolution for the main entry point can be deprecated in a // future major. // Module._findPath is monkey-patchable here. + const { Module, toRealPath } = require('internal/modules/cjs/loader'); let mainPath = Module._findPath(path.resolve(main), null, true); if (!mainPath) return; @@ -37,6 +37,7 @@ function shouldUseESMLoader(mainPath) { const userImports = getOptionValue('--import'); if (userLoaders.length > 0 || userImports.length > 0) return true; + const { readPackageScope } = require('internal/modules/cjs/loader'); // Determine the module format of the main if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) return true; @@ -79,6 +80,7 @@ function executeUserEntryPoint(main = process.argv[1]) { runMainESM(resolvedMain || main); } else { // Module._load is the monkey-patchable CJS module loader. + const { Module } = require('internal/modules/cjs/loader'); Module._load(main, null, true); } } From ae5d36c543f96f2b978c53d500da8d7d707e4218 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:58:06 +0100 Subject: [PATCH 07/15] modules: move callbacks and conditions into modules/esm/utils.js This moves the following utils into modules/esm/utils.js: - Code related to default conditions - The callbackMap (which is now created in the module instead of hanging off the module_wrap binding, since the C++ land does not need it). - Per-isolate module callbacks These are self-contained code that can be included into the built-in snapshot. --- .../modules/esm/create_dynamic_module.js | 6 +- lib/internal/modules/esm/loader.js | 12 +- lib/internal/modules/esm/resolve.js | 32 +----- lib/internal/modules/esm/translators.js | 3 +- lib/internal/modules/esm/utils.js | 107 ++++++++++++++++++ lib/internal/process/esm_loader.js | 29 ----- lib/internal/process/pre_execution.js | 15 +-- lib/internal/vm.js | 7 +- lib/internal/vm/module.js | 4 +- lib/vm.js | 7 +- test/parallel/test-bootstrap-modules.js | 1 + 11 files changed, 132 insertions(+), 91 deletions(-) create mode 100644 lib/internal/modules/esm/utils.js diff --git a/lib/internal/modules/esm/create_dynamic_module.js b/lib/internal/modules/esm/create_dynamic_module.js index f7c20083b6c918..1abb909f58526b 100644 --- a/lib/internal/modules/esm/create_dynamic_module.js +++ b/lib/internal/modules/esm/create_dynamic_module.js @@ -35,7 +35,7 @@ ${ArrayPrototypeJoin(ArrayPrototypeMap(imports, createImport), '\n')} ${ArrayPrototypeJoin(ArrayPrototypeMap(exports, createExport), '\n')} import.meta.done(); `; - const { ModuleWrap, callbackMap } = internalBinding('module_wrap'); + const { ModuleWrap } = internalBinding('module_wrap'); const m = new ModuleWrap(`${url}`, undefined, source, 0, 0); const readyfns = new SafeSet(); @@ -46,8 +46,8 @@ import.meta.done(); if (imports.length) reflect.imports = ObjectCreate(null); - - callbackMap.set(m, { + const { setCallbackForWrap } = require('internal/modules/esm/utils'); + setCallbackForWrap(m, { initializeImportMeta: (meta, wrap) => { meta.exports = reflect.exports; if (reflect.imports) diff --git a/lib/internal/modules/esm/loader.js b/lib/internal/modules/esm/loader.js index c30cc13756cdf4..9a422e8f70f1ce 100644 --- a/lib/internal/modules/esm/loader.js +++ b/lib/internal/modules/esm/loader.js @@ -47,9 +47,12 @@ function newModuleMap() { const { defaultResolve, - DEFAULT_CONDITIONS, } = require('internal/modules/esm/resolve'); +const { + getDefaultConditions, +} = require('internal/modules/esm/utils'); + function getTranslators() { const { translators } = require('internal/modules/esm/translators'); return translators; @@ -363,9 +366,10 @@ class ESMLoader { url = pathToFileURL(`${process.cwd()}/[eval${++this.evalIndex}]`).href ) { const evalInstance = (url) => { - const { ModuleWrap, callbackMap } = internalBinding('module_wrap'); + const { ModuleWrap } = internalBinding('module_wrap'); + const { setCallbackForWrap } = require('internal/modules/esm/utils'); const module = new ModuleWrap(url, undefined, source, 0, 0); - callbackMap.set(module, { + setCallbackForWrap(module, { importModuleDynamically: (specifier, { url }, importAssertions) => { return this.import(specifier, url, importAssertions); } @@ -798,7 +802,7 @@ class ESMLoader { } const chain = this.#hooks.resolve; const context = { - conditions: DEFAULT_CONDITIONS, + conditions: getDefaultConditions(), importAssertions, parentURL, }; diff --git a/lib/internal/modules/esm/resolve.js b/lib/internal/modules/esm/resolve.js index e4ad685a6938e7..aa979eec36fe8b 100644 --- a/lib/internal/modules/esm/resolve.js +++ b/lib/internal/modules/esm/resolve.js @@ -6,7 +6,6 @@ const { ArrayPrototypeJoin, ArrayPrototypeShift, JSONStringify, - ObjectFreeze, ObjectGetOwnPropertyNames, ObjectPrototypeHasOwnProperty, RegExp, @@ -45,7 +44,6 @@ const typeFlag = getOptionValue('--input-type'); const { URL, pathToFileURL, fileURLToPath } = require('internal/url'); const { ERR_INPUT_TYPE_NOT_ALLOWED, - ERR_INVALID_ARG_VALUE, ERR_INVALID_MODULE_SPECIFIER, ERR_INVALID_PACKAGE_CONFIG, ERR_INVALID_PACKAGE_TARGET, @@ -60,25 +58,13 @@ const { const { Module: CJSModule } = require('internal/modules/cjs/loader'); const { getPackageConfig, getPackageScopeConfig } = require('internal/modules/esm/package_config'); +const { getConditionsSet } = require('internal/modules/esm/utils'); /** * @typedef {import('internal/modules/esm/package_config.js').PackageConfig} PackageConfig */ -const userConditions = getOptionValue('--conditions'); -const noAddons = getOptionValue('--no-addons'); -const addonConditions = noAddons ? [] : ['node-addons']; - -const DEFAULT_CONDITIONS = ObjectFreeze([ - 'node', - 'import', - ...addonConditions, - ...userConditions, -]); - -const DEFAULT_CONDITIONS_SET = new SafeSet(DEFAULT_CONDITIONS); - const emittedPackageWarnings = new SafeSet(); function emitTrailingSlashPatternDeprecation(match, pjsonUrl, base) { @@ -147,21 +133,6 @@ function emitLegacyIndexDeprecation(url, packageJSONUrl, base, main) { ); } -/** - * @param {string[]} [conditions] - * @returns {Set} - */ -function getConditionsSet(conditions) { - if (conditions !== undefined && conditions !== DEFAULT_CONDITIONS) { - if (!ArrayIsArray(conditions)) { - throw new ERR_INVALID_ARG_VALUE('conditions', conditions, - 'expected an array'); - } - return new SafeSet(conditions); - } - return DEFAULT_CONDITIONS_SET; -} - const realpathCache = new SafeMap(); /** @@ -1125,7 +1096,6 @@ async function defaultResolve(specifier, context = {}) { } module.exports = { - DEFAULT_CONDITIONS, defaultResolve, encodedSepRegEx, getPackageScopeConfig, diff --git a/lib/internal/modules/esm/translators.js b/lib/internal/modules/esm/translators.js index 2007b8d5ab063d..b21c3c18cf7383 100644 --- a/lib/internal/modules/esm/translators.js +++ b/lib/internal/modules/esm/translators.js @@ -115,7 +115,8 @@ translators.set('module', async function moduleStrategy(url, source, isMain) { maybeCacheSourceMap(url, source); debug(`Translating StandardModule ${url}`); const module = new ModuleWrap(url, undefined, source, 0, 0); - moduleWrap.callbackMap.set(module, { + const { setCallbackForWrap } = require('internal/modules/esm/utils'); + setCallbackForWrap(module, { initializeImportMeta: (meta, wrap) => this.importMetaInitialize(meta, { url }), importModuleDynamically, }); diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js new file mode 100644 index 00000000000000..d92c2f8ad41ea4 --- /dev/null +++ b/lib/internal/modules/esm/utils.js @@ -0,0 +1,107 @@ +'use strict'; +const { + ArrayIsArray, + SafeSet, + SafeWeakMap, + ObjectFreeze, +} = primordials; + +const { + ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING, + ERR_INVALID_ARG_VALUE, +} = require('internal/errors').codes; + +const { getOptionValue } = require('internal/options'); + +const { + setImportModuleDynamicallyCallback, + setInitializeImportMetaObjectCallback +} = internalBinding('module_wrap'); +const { + getModuleFromWrap, +} = require('internal/vm/module'); + +const callbackMap = new SafeWeakMap(); +function setCallbackForWrap(wrap, data) { + callbackMap.set(wrap, data); +} + +let defaultConditions; +function getDefaultConditions() { + if (defaultConditions === undefined) { + initializeDefaultConditions(); + } + return defaultConditions; +} + +let defaultConditionsSet; +function getDefaultConditionsSet() { + if (defaultConditionsSet === undefined) { + initializeDefaultConditions(); + } + return defaultConditionsSet; +} + +function initializeDefaultConditions() { + const userConditions = getOptionValue('--conditions'); + const noAddons = getOptionValue('--no-addons'); + const addonConditions = noAddons ? [] : ['node-addons']; + + defaultConditions = ObjectFreeze([ + 'node', + 'import', + ...addonConditions, + ...userConditions, + ]); + defaultConditionsSet = new SafeSet(defaultConditions); +} + +/** + * @param {string[]} [conditions] + * @returns {Set} + */ +function getConditionsSet(conditions) { + if (conditions !== undefined && conditions !== getDefaultConditions()) { + if (!ArrayIsArray(conditions)) { + throw new ERR_INVALID_ARG_VALUE('conditions', conditions, + 'expected an array'); + } + return new SafeSet(conditions); + } + return getDefaultConditionsSet(); +} + +function initializeImportMetaObject(wrap, meta) { + if (callbackMap.has(wrap)) { + const { initializeImportMeta } = callbackMap.get(wrap); + if (initializeImportMeta !== undefined) { + initializeImportMeta(meta, getModuleFromWrap(wrap) || wrap); + } + } +} + +async function importModuleDynamicallyCallback(wrap, specifier, assertions) { + if (callbackMap.has(wrap)) { + const { importModuleDynamically } = callbackMap.get(wrap); + if (importModuleDynamically !== undefined) { + return importModuleDynamically( + specifier, getModuleFromWrap(wrap) || wrap, assertions); + } + } + throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); +} + +function initializeESM() { + initializeDefaultConditions(); + // Setup per-isolate callbacks that locate data or callbacks that we keep + // track of for different ESM modules. + setInitializeImportMetaObjectCallback(initializeImportMetaObject); + setImportModuleDynamicallyCallback(importModuleDynamicallyCallback); +} + +module.exports = { + setCallbackForWrap, + initializeESM, + getDefaultConditions, + getConditionsSet, +}; diff --git a/lib/internal/process/esm_loader.js b/lib/internal/process/esm_loader.js index 473cea48cafe64..25a2b7191cb5d4 100644 --- a/lib/internal/process/esm_loader.js +++ b/lib/internal/process/esm_loader.js @@ -5,40 +5,11 @@ const { ObjectCreate, } = primordials; -const { - ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING, -} = require('internal/errors').codes; const { ESMLoader } = require('internal/modules/esm/loader'); const { hasUncaughtExceptionCaptureCallback, } = require('internal/process/execution'); const { pathToFileURL } = require('internal/url'); -const { - getModuleFromWrap, -} = require('internal/vm/module'); - -exports.initializeImportMetaObject = function(wrap, meta) { - const { callbackMap } = internalBinding('module_wrap'); - if (callbackMap.has(wrap)) { - const { initializeImportMeta } = callbackMap.get(wrap); - if (initializeImportMeta !== undefined) { - initializeImportMeta(meta, getModuleFromWrap(wrap) || wrap); - } - } -}; - -exports.importModuleDynamicallyCallback = -async function importModuleDynamicallyCallback(wrap, specifier, assertions) { - const { callbackMap } = internalBinding('module_wrap'); - if (callbackMap.has(wrap)) { - const { importModuleDynamically } = callbackMap.get(wrap); - if (importModuleDynamically !== undefined) { - return importModuleDynamically( - specifier, getModuleFromWrap(wrap) || wrap, assertions); - } - } - throw new ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING(); -}; const esmLoader = new ESMLoader(); exports.esmLoader = esmLoader; diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index d808f65424c2de..899a69a6f3bfff 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -6,7 +6,6 @@ const { ObjectDefineProperty, ObjectGetOwnPropertyDescriptor, SafeMap, - SafeWeakMap, StringPrototypeStartsWith, globalThis, } = primordials; @@ -571,20 +570,10 @@ function initializeCJSLoader() { } function initializeESMLoader() { - // Create this WeakMap in js-land because V8 has no C++ API for WeakMap. - internalBinding('module_wrap').callbackMap = new SafeWeakMap(); - if (getEmbedderOptions().shouldNotRegisterESMLoader) return; - const { - setImportModuleDynamicallyCallback, - setInitializeImportMetaObjectCallback - } = internalBinding('module_wrap'); - const esm = require('internal/process/esm_loader'); - // Setup per-isolate callbacks that locate data or callbacks that we keep - // track of for different ESM modules. - setInitializeImportMetaObjectCallback(esm.initializeImportMetaObject); - setImportModuleDynamicallyCallback(esm.importModuleDynamicallyCallback); + const { initializeESM } = require('internal/modules/esm/utils'); + initializeESM(); // Patch the vm module when --experimental-vm-modules is on. // Please update the comments in vm.js when this block changes. diff --git a/lib/internal/vm.js b/lib/internal/vm.js index 32bcdf06234b67..a813c55d9e3aad 100644 --- a/lib/internal/vm.js +++ b/lib/internal/vm.js @@ -94,12 +94,11 @@ function internalCompileFunction(code, params, options) { if (importModuleDynamically !== undefined) { validateFunction(importModuleDynamically, 'options.importModuleDynamically'); - const { importModuleDynamicallyWrap } = - require('internal/vm/module'); - const { callbackMap } = internalBinding('module_wrap'); + const { importModuleDynamicallyWrap } = require('internal/vm/module'); const wrapped = importModuleDynamicallyWrap(importModuleDynamically); const func = result.function; - callbackMap.set(result.cacheKey, { + const { setCallbackForWrap } = require('internal/modules/esm/utils'); + setCallbackForWrap(result.cacheKey, { importModuleDynamically: (s, _k, i) => wrapped(s, func, i), }); } diff --git a/lib/internal/vm/module.js b/lib/internal/vm/module.js index 7e0131c7db2872..91d489fc4c7cdd 100644 --- a/lib/internal/vm/module.js +++ b/lib/internal/vm/module.js @@ -125,8 +125,8 @@ class Module { this[kWrap] = new ModuleWrap(identifier, context, sourceText, options.lineOffset, options.columnOffset, options.cachedData); - - binding.callbackMap.set(this[kWrap], { + const { setCallbackForWrap } = require('internal/modules/esm/utils'); + setCallbackForWrap(this[kWrap], { initializeImportMeta: options.initializeImportMeta, importModuleDynamically: options.importModuleDynamically ? importModuleDynamicallyWrap(options.importModuleDynamically) : diff --git a/lib/vm.js b/lib/vm.js index ec91c8a2651775..dea012156628b6 100644 --- a/lib/vm.js +++ b/lib/vm.js @@ -111,10 +111,9 @@ class Script extends ContextifyScript { if (importModuleDynamically !== undefined) { validateFunction(importModuleDynamically, 'options.importModuleDynamically'); - const { importModuleDynamicallyWrap } = - require('internal/vm/module'); - const { callbackMap } = internalBinding('module_wrap'); - callbackMap.set(this, { + const { importModuleDynamicallyWrap } = require('internal/vm/module'); + const { setCallbackForWrap } = require('internal/modules/esm/utils'); + setCallbackForWrap(this, { importModuleDynamically: importModuleDynamicallyWrap(importModuleDynamically), }); diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 064a208eb7b56f..3ecc7f91a64813 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -65,6 +65,7 @@ const expectedModules = new Set([ 'NativeModule internal/modules/esm/package_config', 'NativeModule internal/modules/esm/resolve', 'NativeModule internal/modules/esm/translators', + 'NativeModule internal/modules/esm/utils', 'NativeModule internal/modules/package_json_reader', 'NativeModule internal/modules/run_main', 'NativeModule internal/net', From 203164bb23931266ac9835aa0d86d05e72654e53 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 23:51:05 +0100 Subject: [PATCH 08/15] bootstrap: make CJS loader snapshotable This patch makes the top-level access to runtime states in the CJS loader lazy, and move the side-effects into a initializeCJS() function that gets called during pre-execution. As a result the CJS loader can be included into the built-in snapshot. --- lib/internal/modules/cjs/loader.js | 187 ++++++++++-------- lib/internal/modules/helpers.js | 34 ++-- lib/internal/process/pre_execution.js | 9 +- ...ction.js => inspector-global-function.mjs} | 0 test/parallel/test-bootstrap-modules.js | 23 +-- .../test-inspector-break-when-eval.js | 7 +- 6 files changed, 136 insertions(+), 124 deletions(-) rename test/fixtures/{inspector-global-function.js => inspector-global-function.mjs} (100%) diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index 466c160506e669..dcb19cdf278bf7 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -72,7 +72,8 @@ const cjsParseCache = new SafeWeakMap(); // Set first due to cycle with ESM loader functions. module.exports = { wrapSafe, Module, toRealPath, readPackageScope, cjsParseCache, - get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; } + get hasLoadedAnyUserCJSModule() { return hasLoadedAnyUserCJSModule; }, + initializeCJS, }; const { BuiltinModule } = require('internal/bootstrap/loaders'); @@ -86,8 +87,8 @@ const { kEmptyObject, filterOwnProperties, setOwnProperty, + getLazy, } = require('internal/util'); -const { Script } = require('vm'); const { internalCompileFunction } = require('internal/vm'); const assert = require('internal/assert'); const fs = require('fs'); @@ -97,21 +98,24 @@ const { sep } = path; const { internalModuleStat } = internalBinding('fs'); const { safeGetenv } = internalBinding('credentials'); const { - cjsConditions, + getCjsConditions, + initializeCjsConditions, hasEsmSyntax, loadBuiltinModule, makeRequireFunction, normalizeReferrerURL, stripBOM, } = require('internal/modules/helpers'); -const { getOptionValue } = require('internal/options'); -const preserveSymlinks = getOptionValue('--preserve-symlinks'); -const preserveSymlinksMain = getOptionValue('--preserve-symlinks-main'); -const shouldReportRequiredModules = process.env.WATCH_REPORT_DEPENDENCIES; -// Do not eagerly grab .manifest, it may be in TDZ -const policy = getOptionValue('--experimental-policy') ? - require('internal/process/policy') : - null; +const packageJsonReader = require('internal/modules/package_json_reader'); +const { getOptionValue, getEmbedderOptions } = require('internal/options'); +const policy = getLazy( + () => (getOptionValue('--experimental-policy') ? require('internal/process/policy') : null) +); +const shouldReportRequiredModules = getLazy(() => process.env.WATCH_REPORT_DEPENDENCIES); + +const getCascadedLoader = getLazy( + () => require('internal/process/esm_loader').esmLoader +); // Whether any user-provided CJS modules had been loaded (executed). // Used for internal assertions. @@ -127,7 +131,6 @@ const { setArrowMessage, } = require('internal/errors'); const { validateString } = require('internal/validators'); -const pendingDeprecation = getOptionValue('--pending-deprecation'); const { CHAR_BACKWARD_SLASH, @@ -140,15 +143,7 @@ const { isProxy } = require('internal/util/types'); -const asyncESM = require('internal/process/esm_loader'); -const { enrichCJSError } = require('internal/modules/esm/translators'); const { kEvaluated } = internalBinding('module_wrap'); -const { - encodedSepRegEx, - packageExportsResolve, - packageImportsResolve -} = require('internal/modules/esm/resolve'); - const isWindows = process.platform === 'win32'; const relativeResolveCache = ObjectCreate(null); @@ -190,13 +185,13 @@ function updateChildren(parent, child, scan) { } function reportModuleToWatchMode(filename) { - if (shouldReportRequiredModules && process.send) { + if (shouldReportRequiredModules() && process.send) { process.send({ 'watch:require': [filename] }); } } function reportModuleNotFoundToWatchMode(basePath, extensions) { - if (shouldReportRequiredModules && process.send) { + if (shouldReportRequiredModules() && process.send) { process.send({ 'watch:require': ArrayPrototypeMap(extensions, (ext) => path.resolve(`${basePath}${ext}`)) }); } } @@ -213,22 +208,6 @@ function Module(id = '', parent) { this.children = []; } -const builtinModules = []; -for (const { 0: id, 1: mod } of BuiltinModule.map) { - if (mod.canBeRequiredByUsers && - BuiltinModule.canBeRequiredWithoutScheme(id)) { - ArrayPrototypePush(builtinModules, id); - } -} - -const allBuiltins = new SafeSet( - ArrayPrototypeFlatMap(builtinModules, (bm) => [bm, `node:${bm}`]) -); -BuiltinModule.getSchemeOnlyModuleNames().forEach((builtin) => allBuiltins.add(`node:${builtin}`)); - -ObjectFreeze(builtinModules); -Module.builtinModules = builtinModules; - Module._cache = ObjectCreate(null); Module._pathCache = ObjectCreate(null); Module._extensions = ObjectCreate(null); @@ -297,26 +276,58 @@ function setModuleParent(value) { moduleParentCache.set(this, value); } -ObjectDefineProperty(Module.prototype, 'parent', { - __proto__: null, - get: pendingDeprecation ? deprecate( - getModuleParent, - 'module.parent is deprecated due to accuracy issues. Please use ' + - 'require.main to find program entry point instead.', - 'DEP0144' - ) : getModuleParent, - set: pendingDeprecation ? deprecate( - setModuleParent, - 'module.parent is deprecated due to accuracy issues. Please use ' + - 'require.main to find program entry point instead.', - 'DEP0144' - ) : setModuleParent, -}); - let debug = require('internal/util/debuglog').debuglog('module', (fn) => { debug = fn; }); -Module._debug = deprecate(debug, 'Module._debug is deprecated.', 'DEP0077'); + +const builtinModules = []; +function initializeCJS() { + const pendingDeprecation = getOptionValue('--pending-deprecation'); + ObjectDefineProperty(Module.prototype, 'parent', { + __proto__: null, + get: pendingDeprecation ? deprecate( + getModuleParent, + 'module.parent is deprecated due to accuracy issues. Please use ' + + 'require.main to find program entry point instead.', + 'DEP0144' + ) : getModuleParent, + set: pendingDeprecation ? deprecate( + setModuleParent, + 'module.parent is deprecated due to accuracy issues. Please use ' + + 'require.main to find program entry point instead.', + 'DEP0144' + ) : setModuleParent, + }); + Module._debug = deprecate(debug, 'Module._debug is deprecated.', 'DEP0077'); + + for (const { 0: id, 1: mod } of BuiltinModule.map) { + if (mod.canBeRequiredByUsers && + BuiltinModule.canBeRequiredWithoutScheme(id)) { + ArrayPrototypePush(builtinModules, id); + } + } + + const allBuiltins = new SafeSet( + ArrayPrototypeFlatMap(builtinModules, (bm) => [bm, `node:${bm}`]) + ); + BuiltinModule.getSchemeOnlyModuleNames().forEach((builtin) => allBuiltins.add(`node:${builtin}`)); + ObjectFreeze(builtinModules); + Module.builtinModules = builtinModules; + + Module.isBuiltin = function isBuiltin(moduleName) { + return allBuiltins.has(moduleName); + }; + + initializeCjsConditions(); + + if (!getEmbedderOptions().noGlobalSearchPaths) { + Module._initPaths(); + } + + // TODO(joyeecheung): deprecate this in favor of a proper hook? + Module.runMain = + require('internal/modules/run_main').executeUserEntryPoint; +} // Given a module name, and a list of paths to test, returns the first // matching file in the following precedence. @@ -337,7 +348,6 @@ function readPackage(requestPath) { const existing = packageJsonCache.get(jsonPath); if (existing !== undefined) return existing; - const packageJsonReader = require('internal/modules/package_json_reader'); const result = packageJsonReader.read(jsonPath); const json = result.containsKeys === false ? '{}' : result.string; if (json === undefined) { @@ -440,7 +450,7 @@ const realpathCache = new SafeMap(); function tryFile(requestPath, isMain) { const rc = _stat(requestPath); if (rc !== 0) return; - if (preserveSymlinks && !isMain) { + if (getOptionValue('--preserve-symlinks') && !isMain) { return path.resolve(requestPath); } return toRealPath(requestPath); @@ -511,9 +521,10 @@ function trySelf(parentPath, request) { } try { + const { packageExportsResolve } = require('internal/modules/esm/resolve'); return finalizeEsmResolution(packageExportsResolve( pathToFileURL(pkgPath + '/package.json'), expansion, pkg, - pathToFileURL(parentPath), cjsConditions), parentPath, pkgPath); + pathToFileURL(parentPath), getCjsConditions()), parentPath, pkgPath); } catch (e) { if (e.code === 'ERR_MODULE_NOT_FOUND') throw createEsmNotFoundErr(request, pkgPath + '/package.json'); @@ -535,9 +546,10 @@ function resolveExports(nmPath, request) { const pkg = _readPackage(pkgPath); if (pkg?.exports != null) { try { + const { packageExportsResolve } = require('internal/modules/esm/resolve'); return finalizeEsmResolution(packageExportsResolve( pathToFileURL(pkgPath + '/package.json'), '.' + expansion, pkg, null, - cjsConditions), null, pkgPath); + getCjsConditions()), null, pkgPath); } catch (e) { if (e.code === 'ERR_MODULE_NOT_FOUND') throw createEsmNotFoundErr(request, pkgPath + '/package.json'); @@ -616,19 +628,19 @@ Module._findPath = function(request, paths, isMain) { if (!trailingSlash) { if (rc === 0) { // File. if (!isMain) { - if (preserveSymlinks) { + if (getOptionValue('--preserve-symlinks')) { filename = path.resolve(basePath); } else { filename = toRealPath(basePath); } - } else if (preserveSymlinksMain) { - // For the main module, we use the preserveSymlinksMain flag instead + } else if (getOptionValue('--preserve-symlinks-main')) { + // For the main module, we use the --preserve-symlinks-main flag instead // mainly for backward compatibility, as the preserveSymlinks flag // historically has not applied to the main module. Most likely this // was intended to keep .bin/ binaries working, as following those // symlinks is usually required for the imports in the corresponding // files to resolve; that said, in some use cases following symlinks - // causes bigger problems which is why the preserveSymlinksMain option + // causes bigger problems which is why the --preserve-symlinks-main option // is needed. filename = path.resolve(basePath); } else { @@ -999,9 +1011,10 @@ Module._resolveFilename = function(request, parent, isMain, options) { const pkg = readPackageScope(parentPath) || {}; if (pkg.data?.imports != null) { try { + const { packageImportsResolve } = require('internal/modules/esm/resolve'); return finalizeEsmResolution( packageImportsResolve(request, pathToFileURL(parentPath), - cjsConditions), parentPath, + getCjsConditions()), parentPath, pkg.path); } catch (e) { if (e.code === 'ERR_MODULE_NOT_FOUND') @@ -1043,6 +1056,7 @@ Module._resolveFilename = function(request, parent, isMain, options) { }; function finalizeEsmResolution(resolved, parentPath, pkgPath) { + const { encodedSepRegEx } = require('internal/modules/esm/resolve'); if (RegExpPrototypeExec(encodedSepRegEx, resolved) !== null) throw new ERR_INVALID_MODULE_SPECIFIER( resolved, 'must not include encoded "/" or "\\" characters', parentPath); @@ -1081,14 +1095,14 @@ Module.prototype.load = function(filename) { Module._extensions[extension](this, filename); this.loaded = true; - const esmLoader = asyncESM.esmLoader; + const cascadedLoader = getCascadedLoader(); // Create module entry at load time to snapshot exports correctly const exports = this.exports; // Preemptively cache if ((module?.module === undefined || module.module.getStatus() < kEvaluated) && - !esmLoader.cjsCache.has(this)) - esmLoader.cjsCache.set(this, exports); + !cascadedLoader.cjsCache.has(this)) + cascadedLoader.cjsCache.set(this, exports); }; @@ -1113,17 +1127,20 @@ Module.prototype.require = function(id) { // (needed for setting breakpoint when called with --inspect-brk) let resolvedArgv; let hasPausedEntry = false; - +let Script; function wrapSafe(filename, content, cjsModuleInstance) { if (patched) { const wrapper = Module.wrap(content); + if (Script === undefined) { + ({ Script } = require('vm')); + } const script = new Script(wrapper, { filename, lineOffset: 0, importModuleDynamically: async (specifier, _, importAssertions) => { - const loader = asyncESM.esmLoader; - return loader.import(specifier, normalizeReferrerURL(filename), - importAssertions); + const cascadedLoader = getCascadedLoader(); + return cascadedLoader.import(specifier, normalizeReferrerURL(filename), + importAssertions); }, }); @@ -1147,9 +1164,9 @@ function wrapSafe(filename, content, cjsModuleInstance) { ], { filename, importModuleDynamically(specifier, _, importAssertions) { - const loader = asyncESM.esmLoader; - return loader.import(specifier, normalizeReferrerURL(filename), - importAssertions); + const cascadedLoader = getCascadedLoader(); + return cascadedLoader.import(specifier, normalizeReferrerURL(filename), + importAssertions); }, }); @@ -1160,8 +1177,10 @@ function wrapSafe(filename, content, cjsModuleInstance) { return result.function; } catch (err) { - if (process.mainModule === cjsModuleInstance) + if (process.mainModule === cjsModuleInstance) { + const { enrichCJSError } = require('internal/modules/esm/translators'); enrichCJSError(err, content); + } throw err; } } @@ -1173,10 +1192,10 @@ function wrapSafe(filename, content, cjsModuleInstance) { Module.prototype._compile = function(content, filename) { let moduleURL; let redirects; - if (policy?.manifest) { + if (policy()?.manifest) { moduleURL = pathToFileURL(filename); - redirects = policy.manifest.getDependencyMapper(moduleURL); - policy.manifest.assertIntegrity(moduleURL, content); + redirects = policy().manifest.getDependencyMapper(moduleURL); + policy().manifest.assertIntegrity(moduleURL, content); } const compiledWrapper = wrapSafe(filename, content, this); @@ -1277,9 +1296,9 @@ Module._extensions['.js'] = function(module, filename) { Module._extensions['.json'] = function(module, filename) { const content = fs.readFileSync(filename, 'utf8'); - if (policy?.manifest) { + if (policy()?.manifest) { const moduleURL = pathToFileURL(filename); - policy.manifest.assertIntegrity(moduleURL, content); + policy().manifest.assertIntegrity(moduleURL, content); } try { @@ -1293,10 +1312,10 @@ Module._extensions['.json'] = function(module, filename) { // Native extension for .node Module._extensions['.node'] = function(module, filename) { - if (policy?.manifest) { + if (policy()?.manifest) { const content = fs.readFileSync(filename); const moduleURL = pathToFileURL(filename); - policy.manifest.assertIntegrity(moduleURL, content); + policy().manifest.assertIntegrity(moduleURL, content); } // Be aware this doesn't use `content` return process.dlopen(module, path.toNamespacedPath(filename)); @@ -1405,9 +1424,5 @@ Module.syncBuiltinESMExports = function syncBuiltinESMExports() { } }; -Module.isBuiltin = function isBuiltin(moduleName) { - return allBuiltins.has(moduleName); -}; - // Backwards compatibility Module.Module = Module; diff --git a/lib/internal/modules/helpers.js b/lib/internal/modules/helpers.js index fea3fbcf48dc00..49c086279e238d 100644 --- a/lib/internal/modules/helpers.js +++ b/lib/internal/modules/helpers.js @@ -25,22 +25,31 @@ const { pathToFileURL, fileURLToPath, URL } = require('internal/url'); const { getOptionValue } = require('internal/options'); const { setOwnProperty } = require('internal/util'); -const userConditions = getOptionValue('--conditions'); let debug = require('internal/util/debuglog').debuglog('module', (fn) => { debug = fn; }); -const noAddons = getOptionValue('--no-addons'); -const addonConditions = noAddons ? [] : ['node-addons']; +let cjsConditions; +function initializeCjsConditions() { + const userConditions = getOptionValue('--conditions'); + const noAddons = getOptionValue('--no-addons'); + const addonConditions = noAddons ? [] : ['node-addons']; + // TODO: Use this set when resolving pkg#exports conditions in loader.js. + cjsConditions = new SafeSet([ + 'require', + 'node', + ...addonConditions, + ...userConditions, + ]); +} -// TODO: Use this set when resolving pkg#exports conditions in loader.js. -const cjsConditions = new SafeSet([ - 'require', - 'node', - ...addonConditions, - ...userConditions, -]); +function getCjsConditions() { + if (cjsConditions === undefined) { + initializeCjsConditions(); + } + return cjsConditions; +} function loadBuiltinModule(filename, request) { const mod = BuiltinModule.map.get(filename); @@ -62,7 +71,7 @@ function makeRequireFunction(mod, redirects) { let require; if (redirects) { const id = mod.filename || mod.id; - const conditions = cjsConditions; + const conditions = getCjsConditions(); const { resolve, reaction } = redirects; require = function require(specifier) { let missing = true; @@ -231,7 +240,8 @@ function hasEsmSyntax(code) { module.exports = { addBuiltinLibsToObject, - cjsConditions, + getCjsConditions, + initializeCjsConditions, hasEsmSyntax, loadBuiltinModule, makeRequireFunction, diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index 899a69a6f3bfff..d66a699b14cec6 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -560,13 +560,8 @@ function initializeWASI() { } function initializeCJSLoader() { - const CJSLoader = require('internal/modules/cjs/loader'); - if (!getEmbedderOptions().noGlobalSearchPaths) { - CJSLoader.Module._initPaths(); - } - // TODO(joyeecheung): deprecate this in favor of a proper hook? - CJSLoader.Module.runMain = - require('internal/modules/run_main').executeUserEntryPoint; + const { initializeCJS } = require('internal/modules/cjs/loader'); + initializeCJS(); } function initializeESMLoader() { diff --git a/test/fixtures/inspector-global-function.js b/test/fixtures/inspector-global-function.mjs similarity index 100% rename from test/fixtures/inspector-global-function.js rename to test/fixtures/inspector-global-function.mjs diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 3ecc7f91a64813..10513e796db719 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -53,26 +53,15 @@ const expectedModules = new Set([ 'NativeModule internal/fs/utils', 'NativeModule internal/idna', 'NativeModule internal/linkedlist', - 'NativeModule internal/modules/helpers', 'NativeModule internal/modules/cjs/loader', - 'NativeModule internal/modules/esm/assert', - 'NativeModule internal/modules/esm/formats', - 'NativeModule internal/modules/esm/get_format', - 'NativeModule internal/modules/esm/initialize_import_meta', - 'NativeModule internal/modules/esm/load', - 'NativeModule internal/modules/esm/loader', - 'NativeModule internal/modules/esm/module_map', - 'NativeModule internal/modules/esm/package_config', - 'NativeModule internal/modules/esm/resolve', - 'NativeModule internal/modules/esm/translators', 'NativeModule internal/modules/esm/utils', + 'NativeModule internal/modules/helpers', 'NativeModule internal/modules/package_json_reader', 'NativeModule internal/modules/run_main', 'NativeModule internal/net', 'NativeModule internal/options', 'NativeModule internal/perf/utils', 'NativeModule internal/priority_queue', - 'NativeModule internal/process/esm_loader', 'NativeModule internal/process/execution', 'NativeModule internal/process/per_thread', 'NativeModule internal/process/pre_execution', @@ -100,15 +89,10 @@ const expectedModules = new Set([ 'NativeModule timers', 'NativeModule url', 'NativeModule util', - 'NativeModule vm', ]); if (!common.isMainThread) { [ - 'Internal Binding messaging', - 'Internal Binding performance', - 'Internal Binding symbols', - 'Internal Binding worker', 'NativeModule diagnostics_channel', 'NativeModule internal/abort_controller', 'NativeModule internal/error_serdes', @@ -139,6 +123,11 @@ if (!common.isMainThread) { ].forEach(expectedModules.add.bind(expectedModules)); } +if (common.isWindows) { + // On Windows fs needs SideEffectFreeRegExpPrototypeExec which uses vm. + expectedModules.add('NativeModule vm'); +} + if (common.hasIntl) { expectedModules.add('Internal Binding icu'); } else { diff --git a/test/sequential/test-inspector-break-when-eval.js b/test/sequential/test-inspector-break-when-eval.js index 1e7ab513dadbbb..f8742790acd74f 100644 --- a/test/sequential/test-inspector-break-when-eval.js +++ b/test/sequential/test-inspector-break-when-eval.js @@ -6,7 +6,10 @@ const { NodeInstance } = require('../common/inspector-helper.js'); const fixtures = require('../common/fixtures'); const { pathToFileURL } = require('url'); -const script = fixtures.path('inspector-global-function.js'); +// This needs to be a ESM to ensure that internal modules are loaded +// before pausing. See +// https://bugs.chromium.org/p/chromium/issues/detail?id=1246905 +const script = fixtures.path('inspector-global-function.mjs'); async function setupDebugger(session) { console.log('[test]', 'Setting up a debugger'); @@ -23,7 +26,7 @@ async function setupDebugger(session) { // NOTE(mmarchini): We wait for the second console.log to ensure we loaded // every internal module before pausing. See - // https://bugs.chromium.org/p/v8/issues/detail?id=10287. + // https://bugs.chromium.org/p/chromium/issues/detail?id=1246905 const waitForReady = session.waitForConsoleOutput('log', 'Ready!'); session.send({ 'method': 'Debugger.resume' }); await waitForReady; From f3b36524aa1240e4d99f40d72c24cf913af6bc12 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 13 Dec 2022 22:14:59 +0100 Subject: [PATCH 09/15] bootstrap: optimize modules loaded in the built-in snapshot Preload essential modules and lazy-load non-essential ones. After this patch, all modules listed by running this snippet: ``` const list = process.moduleLoadList.join('\n'); require('fs').writeSync(1, list, 'utf-8'); ``` (which is roughly the same list as the one in test-bootstrap-module.js for the main thread) are loaded from the snapshot so no additional compilation cost is incurred. --- .../bootstrap/switches/is_main_thread.js | 29 +++++++++++++++++++ lib/internal/process/pre_execution.js | 20 +++++++------ test/parallel/test-bootstrap-modules.js | 3 -- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/lib/internal/bootstrap/switches/is_main_thread.js b/lib/internal/bootstrap/switches/is_main_thread.js index 241729de35cf41..9ba3da2c90e085 100644 --- a/lib/internal/bootstrap/switches/is_main_thread.js +++ b/lib/internal/bootstrap/switches/is_main_thread.js @@ -286,3 +286,32 @@ rawMethods.resetStdioForTesting = function() { stdout = undefined; stderr = undefined; }; + +// Needed by the module loader and generally needed everywhere. +require('fs'); +require('util'); +require('url'); + +require('internal/modules/cjs/loader'); +require('internal/modules/esm/utils'); +require('internal/vm/module'); +// Needed to refresh the time origin. +require('internal/perf/utils'); +// Needed to register the async hooks. +if (internalBinding('config').hasInspector) { + require('internal/inspector_async_hook'); +} +// Needed to set the wasm web API callbacks. +internalBinding('wasm_web_api'); +// Needed to detect whether it's on main thread. +internalBinding('worker'); +// Needed to setup source maps. +require('internal/source_map/source_map_cache'); +// Needed by most execution modes. +require('internal/modules/run_main'); +// Needed to refresh DNS configurations. +require('internal/dns/utils'); +// Needed by almost all execution modes. It's fine to +// load them into the snapshot as long as we don't run +// any of the initialization. +require('internal/process/pre_execution'); diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index d66a699b14cec6..29f8c1d68f1ed8 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -12,7 +12,6 @@ const { const { getOptionValue, - getEmbedderOptions, refreshOptions, } = require('internal/options'); const { reconnectZeroFillToggle } = require('internal/buffer'); @@ -21,6 +20,7 @@ const { exposeInterface, exposeLazyInterfaces, defineReplaceableLazyAttribute, + getLazy, } = require('internal/util'); const { @@ -81,6 +81,7 @@ function prepareExecution(options) { initializeSourceMapsHandlers(); initializeDeprecations(); initializeWASI(); + require('internal/dns/utils').initializeDns(); if (isMainThread) { @@ -263,8 +264,10 @@ function setupFetch() { }); // The WebAssembly Web API: https://webassembly.github.io/spec/web-api - const { wasmStreamingCallback } = require('internal/wasm_web_api'); - internalBinding('wasm_web_api').setImplementation(wasmStreamingCallback); + const lazyWasmStreamingCallback = getLazy(() => require('internal/wasm_web_api').wasmStreamingCallback); + internalBinding('wasm_web_api').setImplementation((streamState, source) => { + lazyWasmStreamingCallback()(streamState, source); + }); } // TODO(aduh95): move this to internal/bootstrap/browser when the CLI flag is @@ -337,12 +340,12 @@ function setupStacktracePrinterOnSigint() { } function initializeReport() { - const { report } = require('internal/process/report'); ObjectDefineProperty(process, 'report', { __proto__: null, enumerable: true, configurable: true, get() { + const { report } = require('internal/process/report'); return report; } }); @@ -357,9 +360,10 @@ function setupDebugEnv() { // This has to be called after initializeReport() is called function initializeReportSignalHandlers() { - const { addSignalHandler } = require('internal/process/report'); - - addSignalHandler(); + if (getOptionValue('--report-on-signal')) { + const { addSignalHandler } = require('internal/process/report'); + addSignalHandler(); + } } function initializeHeapSnapshotSignalHandlers() { @@ -565,8 +569,6 @@ function initializeCJSLoader() { } function initializeESMLoader() { - if (getEmbedderOptions().shouldNotRegisterESMLoader) return; - const { initializeESM } = require('internal/modules/esm/utils'); initializeESM(); diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 10513e796db719..693fa9efb4111b 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -25,7 +25,6 @@ const expectedModules = new Set([ 'Internal Binding options', 'Internal Binding performance', 'Internal Binding process_methods', - 'Internal Binding report', 'Internal Binding string_decoder', 'Internal Binding symbols', 'Internal Binding task_queue', @@ -66,7 +65,6 @@ const expectedModules = new Set([ 'NativeModule internal/process/per_thread', 'NativeModule internal/process/pre_execution', 'NativeModule internal/process/promises', - 'NativeModule internal/process/report', 'NativeModule internal/process/signal', 'NativeModule internal/process/task_queues', 'NativeModule internal/process/warning', @@ -82,7 +80,6 @@ const expectedModules = new Set([ 'NativeModule internal/validators', 'NativeModule internal/vm', 'NativeModule internal/vm/module', - 'NativeModule internal/wasm_web_api', 'NativeModule internal/worker/js_transferable', 'NativeModule path', 'NativeModule querystring', From 8d577325980a9bed8ece57783b31bb622166910d Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 16:47:56 +0100 Subject: [PATCH 10/15] fixup! lib: add getLazy() method to internal/util --- lib/internal/util.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/internal/util.js b/lib/internal/util.js index ecb88334ef4a82..245fcbc0d0b927 100644 --- a/lib/internal/util.js +++ b/lib/internal/util.js @@ -655,16 +655,17 @@ function isArrayBufferDetached(value) { } /** - * @template T return value of loader - * @param {()=>T} loader + * Helper function to lazy-load an initialize-once value. + * @template T Return value of initializer + * @param {()=>T} initializer Initializer of the lazily loaded value. * @returns {()=>T} */ -function getLazy(loader) { +function getLazy(initializer) { let value; let initialized = false; return function() { if (initialized === false) { - value = loader(); + value = initializer(); initialized = true; } return value; From 0c7011adb912e9b1b9b28696e2be3568cfa043a7 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 16:54:15 +0100 Subject: [PATCH 11/15] fixup! modules: move callbacks and conditions into modules/esm/utils.js --- lib/internal/modules/esm/utils.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/internal/modules/esm/utils.js b/lib/internal/modules/esm/utils.js index d92c2f8ad41ea4..2927326097a152 100644 --- a/lib/internal/modules/esm/utils.js +++ b/lib/internal/modules/esm/utils.js @@ -20,6 +20,7 @@ const { const { getModuleFromWrap, } = require('internal/vm/module'); +const assert = require('internal/assert'); const callbackMap = new SafeWeakMap(); function setCallbackForWrap(wrap, data) { @@ -28,20 +29,17 @@ function setCallbackForWrap(wrap, data) { let defaultConditions; function getDefaultConditions() { - if (defaultConditions === undefined) { - initializeDefaultConditions(); - } + assert(defaultConditions !== undefined); return defaultConditions; } let defaultConditionsSet; function getDefaultConditionsSet() { - if (defaultConditionsSet === undefined) { - initializeDefaultConditions(); - } + assert(defaultConditionsSet !== undefined); return defaultConditionsSet; } +// This function is called during pre-execution, before any user code is run. function initializeDefaultConditions() { const userConditions = getOptionValue('--conditions'); const noAddons = getOptionValue('--no-addons'); From cf4681e3f658fc455057611b31a1029715501f9c Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 16:55:11 +0100 Subject: [PATCH 12/15] fixup! bootstrap: make CJS loader snapshotable --- lib/internal/modules/cjs/loader.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index dcb19cdf278bf7..b9a5cb3489fc91 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -281,6 +281,7 @@ let debug = require('internal/util/debuglog').debuglog('module', (fn) => { }); const builtinModules = []; +// This function is called during pre-execution, before any user code is run. function initializeCJS() { const pendingDeprecation = getOptionValue('--pending-deprecation'); ObjectDefineProperty(Module.prototype, 'parent', { From e8a4b5e1332115598bb53d7efe94341099125457 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 17:41:14 +0100 Subject: [PATCH 13/15] fixup! bootstrap: make CJS loader snapshotable --- lib/internal/modules/cjs/loader.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/internal/modules/cjs/loader.js b/lib/internal/modules/cjs/loader.js index b9a5cb3489fc91..2e0be0e3d8a1f8 100644 --- a/lib/internal/modules/cjs/loader.js +++ b/lib/internal/modules/cjs/loader.js @@ -1193,10 +1193,11 @@ function wrapSafe(filename, content, cjsModuleInstance) { Module.prototype._compile = function(content, filename) { let moduleURL; let redirects; - if (policy()?.manifest) { + const manifest = policy()?.manifest; + if (manifest) { moduleURL = pathToFileURL(filename); - redirects = policy().manifest.getDependencyMapper(moduleURL); - policy().manifest.assertIntegrity(moduleURL, content); + redirects = manifest.getDependencyMapper(moduleURL); + manifest.assertIntegrity(moduleURL, content); } const compiledWrapper = wrapSafe(filename, content, this); @@ -1297,9 +1298,10 @@ Module._extensions['.js'] = function(module, filename) { Module._extensions['.json'] = function(module, filename) { const content = fs.readFileSync(filename, 'utf8'); - if (policy()?.manifest) { + const manifest = policy()?.manifest; + if (manifest) { const moduleURL = pathToFileURL(filename); - policy().manifest.assertIntegrity(moduleURL, content); + manifest.assertIntegrity(moduleURL, content); } try { @@ -1313,10 +1315,11 @@ Module._extensions['.json'] = function(module, filename) { // Native extension for .node Module._extensions['.node'] = function(module, filename) { - if (policy()?.manifest) { + const manifest = policy()?.manifest; + if (manifest) { const content = fs.readFileSync(filename); const moduleURL = pathToFileURL(filename); - policy().manifest.assertIntegrity(moduleURL, content); + manifest.assertIntegrity(moduleURL, content); } // Be aware this doesn't use `content` return process.dlopen(module, path.toNamespacedPath(filename)); From faa65011754c3c1e1ccbc49b0bc928ef67f45bad Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 14 Dec 2022 17:42:09 +0100 Subject: [PATCH 14/15] fixup! bootstrap: make CJS loader snapshotable --- test/sequential/test-inspector-break-when-eval.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sequential/test-inspector-break-when-eval.js b/test/sequential/test-inspector-break-when-eval.js index f8742790acd74f..bd9969e0dcfffd 100644 --- a/test/sequential/test-inspector-break-when-eval.js +++ b/test/sequential/test-inspector-break-when-eval.js @@ -6,8 +6,8 @@ const { NodeInstance } = require('../common/inspector-helper.js'); const fixtures = require('../common/fixtures'); const { pathToFileURL } = require('url'); -// This needs to be a ESM to ensure that internal modules are loaded -// before pausing. See +// This needs to be an ES module file to ensure that internal modules are +// loaded before pausing. See // https://bugs.chromium.org/p/chromium/issues/detail?id=1246905 const script = fixtures.path('inspector-global-function.mjs'); From adcd3d2b1f6537ae65d20ce30de4f7f50beff4d6 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 15 Dec 2022 16:15:17 +0100 Subject: [PATCH 15/15] fixup! bootstrap: optimize modules loaded in the built-in snapshot --- lib/internal/process/pre_execution.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/internal/process/pre_execution.js b/lib/internal/process/pre_execution.js index 29f8c1d68f1ed8..c3e84854ccefdd 100644 --- a/lib/internal/process/pre_execution.js +++ b/lib/internal/process/pre_execution.js @@ -20,7 +20,6 @@ const { exposeInterface, exposeLazyInterfaces, defineReplaceableLazyAttribute, - getLazy, } = require('internal/util'); const { @@ -264,9 +263,8 @@ function setupFetch() { }); // The WebAssembly Web API: https://webassembly.github.io/spec/web-api - const lazyWasmStreamingCallback = getLazy(() => require('internal/wasm_web_api').wasmStreamingCallback); internalBinding('wasm_web_api').setImplementation((streamState, source) => { - lazyWasmStreamingCallback()(streamState, source); + require('internal/wasm_web_api').wasmStreamingCallback(streamState, source); }); }