From 84d26e51db73f2b7118513e3ee0ef638016db497 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Mon, 20 Apr 2020 06:16:58 +0800 Subject: [PATCH 01/13] src: split the main context initialization from Environemnt ctor So that it's possible to create an Environment not yet attached to any V8 context. We'll use this to deserialize the V8 context before attaching an Environment to it. --- src/api/environment.cc | 5 +---- src/env.cc | 51 +++++++++++++++++++++++++++++++----------- src/env.h | 10 +++++++++ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/api/environment.cc b/src/api/environment.cc index 997c9530fc2771..1104d758f63e36 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -356,10 +356,7 @@ Environment* CreateEnvironment( exec_args, flags, thread_id); - if (flags & EnvironmentFlags::kOwnsProcessState) { - env->set_abort_on_uncaught_exception(false); - } - + env->InitializeMainContext(context); #if HAVE_INSPECTOR if (inspector_parent_handle) { env->InitializeInspector( diff --git a/src/env.cc b/src/env.cc index bcaa50bd0129b0..494c1877f4483b 100644 --- a/src/env.cc +++ b/src/env.cc @@ -306,15 +306,15 @@ std::string GetExecPath(const std::vector& argv) { } Environment::Environment(IsolateData* isolate_data, - Local context, + Isolate* isolate, const std::vector& args, const std::vector& exec_args, EnvironmentFlags::Flags flags, ThreadId thread_id) - : isolate_(context->GetIsolate()), + : isolate_(isolate), isolate_data_(isolate_data), - immediate_info_(context->GetIsolate()), - tick_info_(context->GetIsolate()), + immediate_info_(isolate), + tick_info_(isolate), timer_base_(uv_now(isolate_data->event_loop())), exec_argv_(exec_args), argv_(args), @@ -322,12 +322,11 @@ Environment::Environment(IsolateData* isolate_data, should_abort_on_uncaught_toggle_(isolate_, 1), stream_base_state_(isolate_, StreamBase::kNumStreamBaseStateFields), flags_(flags), - thread_id_(thread_id.id == static_cast(-1) ? - AllocateEnvironmentThreadId().id : thread_id.id), - context_(context->GetIsolate(), context) { + thread_id_(thread_id.id == static_cast(-1) + ? AllocateEnvironmentThreadId().id + : thread_id.id) { // We'll be creating new objects so make sure we've entered the context. - HandleScope handle_scope(isolate()); - Context::Scope context_scope(context); + HandleScope handle_scope(isolate); // Set some flags if only kDefaultFlags was passed. This can make API version // transitions easier for embedders. @@ -338,6 +337,8 @@ Environment::Environment(IsolateData* isolate_data, } set_env_vars(per_process::system_environment); + // TODO(joyeecheung): pass Isolate* and env_vars to it instead of the entire + // env enabled_debug_list_.Parse(this); // We create new copies of the per-Environment option sets, so that it is @@ -348,13 +349,15 @@ Environment::Environment(IsolateData* isolate_data, inspector_host_port_.reset( new ExclusiveAccess(options_->debug_options().host_port)); + if (flags & EnvironmentFlags::kOwnsProcessState) { + set_abort_on_uncaught_exception(false); + } + #if HAVE_INSPECTOR // We can only create the inspector agent after having cloned the options. inspector_agent_ = std::make_unique(this); #endif - AssignToContext(context, ContextInfo("")); - static uv_once_t init_once = UV_ONCE_INIT; uv_once(&init_once, InitThreadLocalOnce); uv_key_set(&thread_local_env, this); @@ -367,8 +370,7 @@ Environment::Environment(IsolateData* isolate_data, destroy_async_id_list_.reserve(512); - performance_state_ = - std::make_unique(isolate()); + performance_state_ = std::make_unique(isolate); performance_state_->Mark( performance::NODE_PERFORMANCE_MILESTONE_ENVIRONMENT); performance_state_->Mark(performance::NODE_PERFORMANCE_MILESTONE_NODE_START, @@ -400,6 +402,29 @@ Environment::Environment(IsolateData* isolate_data, async_hooks_.no_force_checks(); } + // This adjusts the return value of base_object_count() so that tests that + // check the count do not have to account for internally created BaseObjects. + initial_base_object_count_ = base_object_count(); +} + +Environment::Environment(IsolateData* isolate_data, + Local context, + const std::vector& args, + const std::vector& exec_args, + EnvironmentFlags::Flags flags, + ThreadId thread_id) + : Environment(isolate_data, + context->GetIsolate(), + args, + exec_args, + flags, + thread_id) { + InitializeMainContext(context); +} + +void Environment::InitializeMainContext(Local context) { + context_.Reset(context->GetIsolate(), context); + AssignToContext(context, ContextInfo("")); // TODO(joyeecheung): deserialize when the snapshot covers the environment // properties. CreateProperties(); diff --git a/src/env.h b/src/env.h index 39bcee9c4895c9..8c1bf71ddfaee4 100644 --- a/src/env.h +++ b/src/env.h @@ -919,6 +919,16 @@ class Environment : public MemoryRetainer { static uv_key_t thread_local_env; static inline Environment* GetThreadLocalEnv(); + // Create an Environment without initializing a main Context. Use + // InitializeMainContext() to initialize a main context for it. + Environment(IsolateData* isolate_data, + v8::Isolate* isolate, + const std::vector& args, + const std::vector& exec_args, + EnvironmentFlags::Flags flags, + ThreadId thread_id); + void InitializeMainContext(v8::Local context); + // Create an Environment and initialize the provided main context for it. Environment(IsolateData* isolate_data, v8::Local context, const std::vector& args, From 5eed671e5bf24ab9d4530554c2f4f6bfd14dbe15 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 20 Apr 2019 21:35:27 +0800 Subject: [PATCH 02/13] src: add an ExternalReferenceRegistry class Add an ExternalReferenceRegistry class for registering static external references. To register the external JS to C++ references created in a binding (e.g. when a FunctionTemplate is created): - Add the binding name (same as the id used for `internalBinding()` and `NODE_MODULE_CONTEXT_AWARE_INTERNAL`) to `EXTERNAL_REFERENCE_BINDING_LIST` in `src/node_external_reference.h`. - In the file where the binding is implemented, create a registration function to register the static C++ references (e.g. the C++ functions in `v8::FunctionCallback` associated with the function templates), like this: ```c++ void RegisterExternalReferences( ExternalReferenceRegistry* registry) { registry->Register(cpp_func_1); } ``` - At the end of the file where `NODE_MODULE_CONTEXT_AWARE_INTERNAL` is also usually called, register the registration function with ``` NODE_MODULE_EXTERNAL_REFERENCE(binding_name, RegisterExternalReferences); ``` --- node.gyp | 2 + src/node.cc | 1 + src/node_external_reference.cc | 22 ++++++++++ src/node_external_reference.h | 67 ++++++++++++++++++++++++++++++ src/node_main_instance.cc | 15 ++++++- src/node_main_instance.h | 4 ++ tools/snapshot/snapshot_builder.cc | 8 ++-- 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 src/node_external_reference.cc create mode 100644 src/node_external_reference.h diff --git a/node.gyp b/node.gyp index da6745e1c5f00a..9153599f25b558 100644 --- a/node.gyp +++ b/node.gyp @@ -594,6 +594,7 @@ 'src/node_dir.cc', 'src/node_env_var.cc', 'src/node_errors.cc', + 'src/node_external_reference.cc', 'src/node_file.cc', 'src/node_http_parser.cc', 'src/node_http2.cc', @@ -688,6 +689,7 @@ 'src/node_contextify.h', 'src/node_dir.h', 'src/node_errors.h', + 'src/node_external_reference.h', 'src/node_file.h', 'src/node_file-inl.h', 'src/node_http_common.h', diff --git a/src/node.cc b/src/node.cc index eabee549a68b61..e27edec0efe7d3 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1069,6 +1069,7 @@ int Start(int argc, char** argv) { if (blob != nullptr) { // TODO(joyeecheung): collect external references and set it in // params.external_references. + external_references = NodeMainInstance::CollectExternalReferences(); external_references.push_back(reinterpret_cast(nullptr)); params.external_references = external_references.data(); params.snapshot_blob = blob; diff --git a/src/node_external_reference.cc b/src/node_external_reference.cc new file mode 100644 index 00000000000000..73e1489865d3a4 --- /dev/null +++ b/src/node_external_reference.cc @@ -0,0 +1,22 @@ +#include "node_external_reference.h" +#include +#include +#include "util.h" + +namespace node { + +const std::vector& ExternalReferenceRegistry::external_references() { + CHECK(!is_finalized_); + external_references_.push_back(reinterpret_cast(nullptr)); + is_finalized_ = true; + return external_references_; +} + +ExternalReferenceRegistry::ExternalReferenceRegistry() { +#define V(modname) _register_external_reference_##modname(this); + EXTERNAL_REFERENCE_BINDING_LIST(V) +#undef V + // TODO(joyeecheung): collect more external references here. +} + +} // namespace node diff --git a/src/node_external_reference.h b/src/node_external_reference.h new file mode 100644 index 00000000000000..544236e3af669c --- /dev/null +++ b/src/node_external_reference.h @@ -0,0 +1,67 @@ +#ifndef SRC_NODE_EXTERNAL_REFERENCE_H_ +#define SRC_NODE_EXTERNAL_REFERENCE_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include +#include +#include "v8.h" + +namespace node { + +// This class manages the external references from the V8 heap +// to the C++ addresses in Node.js. +class ExternalReferenceRegistry { + public: + ExternalReferenceRegistry(); + +#define ALLOWED_EXTERNAL_REFERENCE_TYPES(V) \ + V(v8::FunctionCallback) \ + V(v8::AccessorGetterCallback) \ + V(v8::AccessorSetterCallback) \ + V(v8::AccessorNameGetterCallback) \ + V(v8::AccessorNameSetterCallback) \ + V(v8::GenericNamedPropertyDefinerCallback) \ + V(v8::GenericNamedPropertyDeleterCallback) \ + V(v8::GenericNamedPropertyEnumeratorCallback) \ + V(v8::GenericNamedPropertyQueryCallback) \ + V(v8::GenericNamedPropertySetterCallback) + +#define V(ExternalReferenceType) \ + void Register(ExternalReferenceType addr) { RegisterT(addr); } + ALLOWED_EXTERNAL_REFERENCE_TYPES(V) +#undef V + + // This can be called only once. + const std::vector& external_references(); + + bool is_empty() { return external_references_.empty(); } + + private: + template + void RegisterT(T* address) { + external_references_.push_back(reinterpret_cast(address)); + } + bool is_finalized_ = false; + std::vector external_references_; +}; + +#define EXTERNAL_REFERENCE_BINDING_LIST(V) + +} // namespace node + +// Declare all the external reference registration functions here, +// and define them later with #NODE_MODULE_EXTERNAL_REFERENCE(modname, func); +#define V(modname) \ + void _register_external_reference_##modname( \ + node::ExternalReferenceRegistry* registry); +EXTERNAL_REFERENCE_BINDING_LIST(V) +#undef V + +#define NODE_MODULE_EXTERNAL_REFERENCE(modname, func) \ + void _register_external_reference_##modname( \ + node::ExternalReferenceRegistry* registry) { \ + func(registry); \ + } +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS +#endif // SRC_NODE_EXTERNAL_REFERENCE_H_ diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index f638e26dba5a04..19f25574acbe95 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -1,7 +1,9 @@ #include -#include "node_main_instance.h" +#include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" +#include "node_main_instance.h" #include "node_options-inl.h" #include "node_v8_platform-inl.h" #include "util-inl.h" @@ -22,6 +24,8 @@ using v8::Local; using v8::Locker; using v8::SealHandleScope; +std::unique_ptr NodeMainInstance::registry_ = + nullptr; NodeMainInstance::NodeMainInstance(Isolate* isolate, uv_loop_t* event_loop, MultiIsolatePlatform* platform, @@ -41,6 +45,15 @@ NodeMainInstance::NodeMainInstance(Isolate* isolate, SetIsolateMiscHandlers(isolate_, {}); } +const std::vector& NodeMainInstance::CollectExternalReferences() { + // Cannot be called more than once. + CHECK_NULL(registry_); + registry_.reset(new ExternalReferenceRegistry()); + + // TODO(joyeecheung): collect more external references here. + return registry_->external_references(); +} + std::unique_ptr NodeMainInstance::Create( Isolate* isolate, uv_loop_t* event_loop, diff --git a/src/node_main_instance.h b/src/node_main_instance.h index b8178c2774e795..de1f4dca2b7cc6 100644 --- a/src/node_main_instance.h +++ b/src/node_main_instance.h @@ -13,6 +13,8 @@ namespace node { +class ExternalReferenceRegistry; + // TODO(joyeecheung): align this with the Worker/WorkerThreadData class. // We may be able to create an abstract class to reuse some of the routines. class NodeMainInstance { @@ -66,6 +68,7 @@ class NodeMainInstance { // snapshot. static const std::vector* GetIsolateDataIndexes(); static v8::StartupData* GetEmbeddedSnapshotBlob(); + static const std::vector& CollectExternalReferences(); static const size_t kNodeContextIndex = 0; NodeMainInstance(const NodeMainInstance&) = delete; @@ -80,6 +83,7 @@ class NodeMainInstance { const std::vector& args, const std::vector& exec_args); + static std::unique_ptr registry_; std::vector args_; std::vector exec_args_; std::unique_ptr array_buffer_allocator_; diff --git a/tools/snapshot/snapshot_builder.cc b/tools/snapshot/snapshot_builder.cc index 8a97513ba905fc..7d292391ab334a 100644 --- a/tools/snapshot/snapshot_builder.cc +++ b/tools/snapshot/snapshot_builder.cc @@ -1,6 +1,8 @@ #include "snapshot_builder.h" #include #include +#include "env-inl.h" +#include "node_external_reference.h" #include "node_internals.h" #include "node_main_instance.h" #include "node_v8_platform-inl.h" @@ -63,10 +65,8 @@ const std::vector* NodeMainInstance::GetIsolateDataIndexes() { std::string SnapshotBuilder::Generate( const std::vector args, const std::vector exec_args) { - // TODO(joyeecheung): collect external references and set it in - // params.external_references. - std::vector external_references = { - reinterpret_cast(nullptr)}; + const std::vector& external_references = + NodeMainInstance::CollectExternalReferences(); Isolate* isolate = Isolate::Allocate(); per_process::v8_platform.Platform()->RegisterIsolate(isolate, uv_default_loop()); From ae47366c9c4b171d7af53c835bf5e25be68e4d1f Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 30 Apr 2020 16:03:59 +0800 Subject: [PATCH 03/13] tools: enable Node.js command line flags in node_mksnapshot Pass the flags down to node_mksnapshot so that we can use them when generating the snapshot (e.g. to debug or enable V8 flags) --- tools/snapshot/node_mksnapshot.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/snapshot/node_mksnapshot.cc b/tools/snapshot/node_mksnapshot.cc index 29f9e1cbcbb096..0ea21990081fea 100644 --- a/tools/snapshot/node_mksnapshot.cc +++ b/tools/snapshot/node_mksnapshot.cc @@ -33,12 +33,18 @@ int main(int argc, char* argv[]) { return 1; } +// Windows needs conversion from wchar_t to char. See node_main.cc +#ifdef _WIN32 int node_argc = 1; char argv0[] = "node"; char* node_argv[] = {argv0, nullptr}; - node::InitializationResult result = node::InitializeOncePerProcess(node_argc, node_argv); +#else + node::InitializationResult result = + node::InitializeOncePerProcess(argc, argv); +#endif + CHECK(!result.early_return); CHECK_EQ(result.exit_code, 0); From d275cb282fc105b5166e3e029cbf29f1fc548f7b Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sun, 21 Apr 2019 12:47:32 +0800 Subject: [PATCH 04/13] src: snapshot Environment upon instantiation This includes the initial Environment (without running bootstrap scripts) into the builtin snapshot --- src/aliased_buffer.h | 66 ++++- src/api/environment.cc | 8 +- src/debug_utils.h | 3 +- src/env-inl.h | 29 --- src/env.cc | 388 +++++++++++++++++++++++++++-- src/env.h | 74 +++++- src/node.cc | 10 +- src/node_external_reference.h | 5 +- src/node_main_instance.cc | 142 +++++++---- src/node_main_instance.h | 6 +- src/node_perf.cc | 50 +++- src/node_perf_common.h | 29 +-- src/node_process_object.cc | 10 +- src/node_snapshot_stub.cc | 4 + src/util.h | 1 + tools/snapshot/snapshot_builder.cc | 60 ++++- 16 files changed, 722 insertions(+), 163 deletions(-) diff --git a/src/aliased_buffer.h b/src/aliased_buffer.h index e762e8ede8ebee..408f158b7880ca 100644 --- a/src/aliased_buffer.h +++ b/src/aliased_buffer.h @@ -4,11 +4,14 @@ #if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS #include +#include #include "util-inl.h" #include "v8.h" namespace node { +typedef size_t AliasedBufferInfo; + /** * Do not use this class directly when creating instances of it - use the * Aliased*Array defined at the end of this file instead. @@ -32,9 +35,15 @@ template ::value>> class AliasedBufferBase { public: - AliasedBufferBase(v8::Isolate* isolate, const size_t count) - : isolate_(isolate), count_(count), byte_offset_(0) { + AliasedBufferBase(v8::Isolate* isolate, + const size_t count, + const AliasedBufferInfo* info = nullptr) + : isolate_(isolate), count_(count), byte_offset_(0), info_(info) { CHECK_GT(count, 0); + if (info != nullptr) { + // Will be deserialized later. + return; + } const v8::HandleScope handle_scope(isolate_); const size_t size_in_bytes = MultiplyWithOverflowCheck(sizeof(NativeT), count); @@ -62,10 +71,17 @@ class AliasedBufferBase { v8::Isolate* isolate, const size_t byte_offset, const size_t count, - const AliasedBufferBase& backing_buffer) - : isolate_(isolate), count_(count), byte_offset_(byte_offset) { + const AliasedBufferBase& backing_buffer, + const AliasedBufferInfo* info = nullptr) + : isolate_(isolate), + count_(count), + byte_offset_(byte_offset), + info_(info) { + if (info != nullptr) { + // Will be deserialized later. + return; + } const v8::HandleScope handle_scope(isolate_); - v8::Local ab = backing_buffer.GetArrayBuffer(); // validate that the byte_offset is aligned with sizeof(NativeT) @@ -86,10 +102,33 @@ class AliasedBufferBase { count_(that.count_), byte_offset_(that.byte_offset_), buffer_(that.buffer_) { + DCHECK_NULL(info_); js_array_ = v8::Global(that.isolate_, that.GetJSArray()); } + AliasedBufferInfo Serialize(v8::Local context, + v8::SnapshotCreator* creator) { + DCHECK_NULL(info_); + return creator->AddData(context, GetJSArray()); + } + + inline void Deserialize(v8::Local context) { + DCHECK_NOT_NULL(info_); + v8::Local arr = + context->GetDataFromSnapshotOnce(*info_).ToLocalChecked(); + // These may not hold true for AliasedBuffers that have grown, so should + // be removed when we expand the snapshot support. + DCHECK_EQ(count_, arr->Length()); + DCHECK_EQ(byte_offset_, arr->ByteOffset()); + uint8_t* raw = + static_cast(arr->Buffer()->GetBackingStore()->Data()); + buffer_ = reinterpret_cast(raw + byte_offset_); + js_array_.Reset(isolate_, arr); + info_ = nullptr; + } + AliasedBufferBase& operator=(AliasedBufferBase&& that) noexcept { + DCHECK_NULL(info_); this->~AliasedBufferBase(); isolate_ = that.isolate_; count_ = that.count_; @@ -155,6 +194,7 @@ class AliasedBufferBase { * Get the underlying v8 TypedArray overlayed on top of the native buffer */ v8::Local GetJSArray() const { + DCHECK_NULL(info_); return js_array_.Get(isolate_); } @@ -171,6 +211,7 @@ class AliasedBufferBase { * through the GetValue/SetValue/operator[] methods */ inline const NativeT* GetNativeBuffer() const { + DCHECK_NULL(info_); return buffer_; } @@ -186,6 +227,7 @@ class AliasedBufferBase { */ inline void SetValue(const size_t index, NativeT value) { DCHECK_LT(index, count_); + DCHECK_NULL(info_); buffer_[index] = value; } @@ -193,6 +235,7 @@ class AliasedBufferBase { * Get value at position index */ inline const NativeT GetValue(const size_t index) const { + DCHECK_NULL(info_); DCHECK_LT(index, count_); return buffer_[index]; } @@ -201,6 +244,7 @@ class AliasedBufferBase { * Effectively, a synonym for GetValue/SetValue */ Reference operator[](size_t index) { + DCHECK_NULL(info_); return Reference(this, index); } @@ -216,6 +260,7 @@ class AliasedBufferBase { // Should only be used on an owning array, not one created as a sub array of // an owning `AliasedBufferBase`. void reserve(size_t new_capacity) { + DCHECK_NULL(info_); DCHECK_GE(new_capacity, count_); DCHECK_EQ(byte_offset_, 0); const v8::HandleScope handle_scope(isolate_); @@ -244,11 +289,14 @@ class AliasedBufferBase { } private: - v8::Isolate* isolate_; - size_t count_; - size_t byte_offset_; - NativeT* buffer_; + v8::Isolate* isolate_ = nullptr; + size_t count_ = 0; + size_t byte_offset_ = 0; + NativeT* buffer_ = nullptr; v8::Global js_array_; + + // Deserialize data + const AliasedBufferInfo* info_ = nullptr; }; typedef AliasedBufferBase AliasedInt32Array; diff --git a/src/api/environment.cc b/src/api/environment.cc index 1104d758f63e36..b288ee7646926d 100644 --- a/src/api/environment.cc +++ b/src/api/environment.cc @@ -350,13 +350,7 @@ Environment* CreateEnvironment( // TODO(addaleax): This is a much better place for parsing per-Environment // options than the global parse call. Environment* env = new Environment( - isolate_data, - context, - args, - exec_args, - flags, - thread_id); - env->InitializeMainContext(context); + isolate_data, context, args, exec_args, nullptr, flags, thread_id); #if HAVE_INSPECTOR if (inspector_parent_handle) { env->InitializeInspector( diff --git a/src/debug_utils.h b/src/debug_utils.h index 845a78c53c30fc..4915fbae325ecc 100644 --- a/src/debug_utils.h +++ b/src/debug_utils.h @@ -46,7 +46,8 @@ void FWrite(FILE* file, const std::string& str); V(INSPECTOR_PROFILER) \ V(CODE_CACHE) \ V(NGTCP2_DEBUG) \ - V(WASI) + V(WASI) \ + V(MKSNAPSHOT) enum class DebugCategory { #define V(name) name, diff --git a/src/env-inl.h b/src/env-inl.h index 20794a3cbe9539..44fa1125a3c455 100644 --- a/src/env-inl.h +++ b/src/env-inl.h @@ -70,29 +70,6 @@ inline v8::Local IsolateData::async_wrap_provider(int index) const { return async_wrap_providers_[index].Get(isolate_); } -inline AsyncHooks::AsyncHooks() - : async_ids_stack_(env()->isolate(), 16 * 2), - fields_(env()->isolate(), kFieldsCount), - async_id_fields_(env()->isolate(), kUidFieldsCount) { - clear_async_id_stack(); - - // Always perform async_hooks checks, not just when async_hooks is enabled. - // TODO(AndreasMadsen): Consider removing this for LTS releases. - // See discussion in https://github.com/nodejs/node/pull/15454 - // When removing this, do it by reverting the commit. Otherwise the test - // and flag changes won't be included. - fields_[kCheck] = 1; - - // kDefaultTriggerAsyncId should be -1, this indicates that there is no - // specified default value and it should fallback to the executionAsyncId. - // 0 is not used as the magic value, because that indicates a missing context - // which is different from a default context. - async_id_fields_[AsyncHooks::kDefaultTriggerAsyncId] = -1; - - // kAsyncIdCounter should start at 1 because that'll be the id the execution - // context during bootstrap (code that runs before entering uv_run()). - async_id_fields_[AsyncHooks::kAsyncIdCounter] = 1; -} inline AliasedUint32Array& AsyncHooks::fields() { return fields_; } @@ -277,9 +254,6 @@ inline void Environment::PopAsyncCallbackScope() { async_callback_scope_depth_--; } -inline ImmediateInfo::ImmediateInfo(v8::Isolate* isolate) - : fields_(isolate, kFieldsCount) {} - inline AliasedUint32Array& ImmediateInfo::fields() { return fields_; } @@ -304,9 +278,6 @@ inline void ImmediateInfo::ref_count_dec(uint32_t decrement) { fields_[kRefCount] -= decrement; } -inline TickInfo::TickInfo(v8::Isolate* isolate) - : fields_(isolate, kFieldsCount) {} - inline AliasedUint8Array& TickInfo::fields() { return fields_; } diff --git a/src/env.cc b/src/env.cc index 494c1877f4483b..dde1c16db12725 100644 --- a/src/env.cc +++ b/src/env.cc @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include namespace node { @@ -309,18 +311,26 @@ Environment::Environment(IsolateData* isolate_data, Isolate* isolate, const std::vector& args, const std::vector& exec_args, + const EnvSerializeInfo* env_info, EnvironmentFlags::Flags flags, ThreadId thread_id) : isolate_(isolate), isolate_data_(isolate_data), - immediate_info_(isolate), - tick_info_(isolate), + async_hooks_(isolate, MAYBE_FIELD_PTR(env_info, async_hooks)), + immediate_info_(isolate, MAYBE_FIELD_PTR(env_info, immediate_info)), + tick_info_(isolate, MAYBE_FIELD_PTR(env_info, tick_info)), timer_base_(uv_now(isolate_data->event_loop())), exec_argv_(exec_args), argv_(args), exec_path_(GetExecPath(args)), - should_abort_on_uncaught_toggle_(isolate_, 1), - stream_base_state_(isolate_, StreamBase::kNumStreamBaseStateFields), + should_abort_on_uncaught_toggle_( + isolate_, + 1, + MAYBE_FIELD_PTR(env_info, should_abort_on_uncaught_toggle)), + stream_base_state_(isolate_, + StreamBase::kNumStreamBaseStateFields, + MAYBE_FIELD_PTR(env_info, stream_base_state)), + environment_start_time_(PERFORMANCE_NOW()), flags_(flags), thread_id_(thread_id.id == static_cast(-1) ? AllocateEnvironmentThreadId().id @@ -370,14 +380,8 @@ Environment::Environment(IsolateData* isolate_data, destroy_async_id_list_.reserve(512); - performance_state_ = std::make_unique(isolate); - performance_state_->Mark( - performance::NODE_PERFORMANCE_MILESTONE_ENVIRONMENT); - performance_state_->Mark(performance::NODE_PERFORMANCE_MILESTONE_NODE_START, - per_process::node_start_time); - performance_state_->Mark( - performance::NODE_PERFORMANCE_MILESTONE_V8_START, - performance::performance_v8_start); + performance_state_ = std::make_unique( + isolate, MAYBE_FIELD_PTR(env_info, performance_state)); if (*TRACE_EVENT_API_GET_CATEGORY_GROUP_ENABLED( TRACING_CATEGORY_NODE1(environment)) != 0) { @@ -395,13 +399,6 @@ Environment::Environment(IsolateData* isolate_data, std::move(traced_value)); } - // By default, always abort when --abort-on-uncaught-exception was passed. - should_abort_on_uncaught_toggle_[0] = 1; - - if (options_->no_force_async_hooks_checks) { - async_hooks_.no_force_checks(); - } - // This adjusts the return value of base_object_count() so that tests that // check the count do not have to account for internally created BaseObjects. initial_base_object_count_ = base_object_count(); @@ -411,23 +408,42 @@ Environment::Environment(IsolateData* isolate_data, Local context, const std::vector& args, const std::vector& exec_args, + const EnvSerializeInfo* env_info, EnvironmentFlags::Flags flags, ThreadId thread_id) : Environment(isolate_data, context->GetIsolate(), args, exec_args, + env_info, flags, thread_id) { - InitializeMainContext(context); + InitializeMainContext(context, env_info); } -void Environment::InitializeMainContext(Local context) { +void Environment::InitializeMainContext(Local context, + const EnvSerializeInfo* env_info) { context_.Reset(context->GetIsolate(), context); AssignToContext(context, ContextInfo("")); - // TODO(joyeecheung): deserialize when the snapshot covers the environment - // properties. - CreateProperties(); + if (env_info != nullptr) { + DeserializeProperties(env_info); + } else { + CreateProperties(); + } + + if (options_->no_force_async_hooks_checks) { + async_hooks_.no_force_checks(); + } + + // By default, always abort when --abort-on-uncaught-exception was passed. + should_abort_on_uncaught_toggle_[0] = 1; + + performance_state_->Mark(performance::NODE_PERFORMANCE_MILESTONE_ENVIRONMENT, + environment_start_time_); + performance_state_->Mark(performance::NODE_PERFORMANCE_MILESTONE_NODE_START, + per_process::node_start_time); + performance_state_->Mark(performance::NODE_PERFORMANCE_MILESTONE_V8_START, + performance::performance_v8_start); // This adjusts the return value of base_object_count() so that tests that // check the count do not have to account for internally created BaseObjects. @@ -474,8 +490,8 @@ Environment::~Environment() { inspector_agent_.reset(); #endif - context()->SetAlignedPointerInEmbedderData( - ContextEmbedderIndex::kEnvironment, nullptr); + context()->SetAlignedPointerInEmbedderData(ContextEmbedderIndex::kEnvironment, + nullptr); if (trace_state_observer_) { tracing::AgentWriterHandle* writer = GetTracingAgentWriter(); @@ -951,14 +967,154 @@ void Environment::CollectUVExceptionInfo(Local object, syscall, message, path, dest); } +ImmediateInfo::ImmediateInfo(v8::Isolate* isolate, const SerializeInfo* info) + : fields_(isolate, kFieldsCount, MAYBE_FIELD_PTR(info, fields)) {} + +ImmediateInfo::SerializeInfo ImmediateInfo::Serialize( + v8::Local context, v8::SnapshotCreator* creator) { + return {fields_.Serialize(context, creator)}; +} + +void ImmediateInfo::Deserialize(Local context) { + fields_.Deserialize(context); +} + +std::ostream& operator<<(std::ostream& output, + const ImmediateInfo::SerializeInfo& i) { + output << "{ " << i.fields << " }"; + return output; +} + void ImmediateInfo::MemoryInfo(MemoryTracker* tracker) const { tracker->TrackField("fields", fields_); } +TickInfo::SerializeInfo TickInfo::Serialize(v8::Local context, + v8::SnapshotCreator* creator) { + return {fields_.Serialize(context, creator)}; +} + +void TickInfo::Deserialize(Local context) { + fields_.Deserialize(context); +} + +std::ostream& operator<<(std::ostream& output, + const TickInfo::SerializeInfo& i) { + output << "{ " << i.fields << " }"; + return output; +} + void TickInfo::MemoryInfo(MemoryTracker* tracker) const { tracker->TrackField("fields", fields_); } +TickInfo::TickInfo(v8::Isolate* isolate, const SerializeInfo* info) + : fields_( + isolate, kFieldsCount, info == nullptr ? nullptr : &(info->fields)) {} + +AsyncHooks::AsyncHooks(v8::Isolate* isolate, const SerializeInfo* info) + : async_ids_stack_(isolate, 16 * 2, MAYBE_FIELD_PTR(info, async_ids_stack)), + fields_(isolate, kFieldsCount, MAYBE_FIELD_PTR(info, fields)), + async_id_fields_( + isolate, kUidFieldsCount, MAYBE_FIELD_PTR(info, async_id_fields)), + info_(info) { + v8::HandleScope handle_scope(isolate); + if (info == nullptr) { + clear_async_id_stack(); + + // Always perform async_hooks checks, not just when async_hooks is enabled. + // TODO(AndreasMadsen): Consider removing this for LTS releases. + // See discussion in https://github.com/nodejs/node/pull/15454 + // When removing this, do it by reverting the commit. Otherwise the test + // and flag changes won't be included. + fields_[kCheck] = 1; + + // kDefaultTriggerAsyncId should be -1, this indicates that there is no + // specified default value and it should fallback to the executionAsyncId. + // 0 is not used as the magic value, because that indicates a missing + // context which is different from a default context. + async_id_fields_[AsyncHooks::kDefaultTriggerAsyncId] = -1; + + // kAsyncIdCounter should start at 1 because that'll be the id the execution + // context during bootstrap (code that runs before entering uv_run()). + async_id_fields_[AsyncHooks::kAsyncIdCounter] = 1; + } +} + +void AsyncHooks::Deserialize(Local context) { + async_ids_stack_.Deserialize(context); + fields_.Deserialize(context); + async_id_fields_.Deserialize(context); + if (info_->js_execution_async_resources != 0) { + v8::Local arr = context + ->GetDataFromSnapshotOnce( + info_->js_execution_async_resources) + .ToLocalChecked(); + js_execution_async_resources_.Reset(context->GetIsolate(), arr); + } + + native_execution_async_resources_.resize( + info_->native_execution_async_resources.size()); + for (size_t i = 0; i < info_->native_execution_async_resources.size(); ++i) { + v8::Local obj = + context + ->GetDataFromSnapshotOnce( + info_->native_execution_async_resources[i]) + .ToLocalChecked(); + native_execution_async_resources_[i].Reset(context->GetIsolate(), obj); + } + info_ = nullptr; +} + +std::ostream& operator<<(std::ostream& output, + const std::vector& v) { + output << "{ "; + for (const SnapshotIndex i : v) { + output << i << ", "; + } + output << " }\n"; + return output; +} + +std::ostream& operator<<(std::ostream& output, + const AsyncHooks::SerializeInfo& i) { + output << "{\n" + << " " << i.async_ids_stack << ", // async_ids_stack\n" + << " " << i.fields << ", // fields\n" + << " " << i.async_id_fields << ", // async_id_fields\n" + << " " << i.js_execution_async_resources + << ", // js_execution_async_resources\n" + << " " << i.native_execution_async_resources + << ", // native_execution_async_resources\n" + << "}"; + return output; +} + +AsyncHooks::SerializeInfo AsyncHooks::Serialize(Local context, + SnapshotCreator* creator) { + SerializeInfo info; + info.async_ids_stack = async_ids_stack_.Serialize(context, creator); + info.fields = fields_.Serialize(context, creator); + info.async_id_fields = async_id_fields_.Serialize(context, creator); + if (!js_execution_async_resources_.IsEmpty()) { + info.js_execution_async_resources = creator->AddData( + context, js_execution_async_resources_.Get(context->GetIsolate())); + CHECK_NE(info.js_execution_async_resources, 0); + } else { + info.js_execution_async_resources = 0; + } + + info.native_execution_async_resources.resize( + native_execution_async_resources_.size()); + for (size_t i = 0; i < native_execution_async_resources_.size(); i++) { + info.native_execution_async_resources[i] = creator->AddData( + context, + native_execution_async_resources_[i].Get(context->GetIsolate())); + } + + return info; +} + void AsyncHooks::MemoryInfo(MemoryTracker* tracker) const { tracker->TrackField("async_ids_stack", async_ids_stack_); tracker->TrackField("fields", fields_); @@ -1030,6 +1186,186 @@ void Environment::RemoveUnmanagedFd(int fd) { } } +void Environment::ForEachBaseObject(BaseObjectIterator iterator) { + size_t i = 0; + for (const auto& hook : cleanup_hooks_) { + BaseObject* obj = hook.GetBaseObject(); + if (obj != nullptr) iterator(i, obj); + i++; + } +} + +void PrintBaseObject(size_t i, BaseObject* obj) { + std::cout << "#" << i << " " << obj << ": " << obj->MemoryInfoName() << "\n"; +} + +void Environment::PrintAllBaseObjects() { + std::cout << "BaseObjects\n"; + ForEachBaseObject(PrintBaseObject); +} + +EnvSerializeInfo Environment::Serialize(SnapshotCreator* creator) { + EnvSerializeInfo info; + Local ctx = context(); + + info.async_hooks = async_hooks_.Serialize(ctx, creator); + info.immediate_info = immediate_info_.Serialize(ctx, creator); + info.tick_info = tick_info_.Serialize(ctx, creator); + info.performance_state = performance_state_->Serialize(ctx, creator); + info.stream_base_state = stream_base_state_.Serialize(ctx, creator); + info.should_abort_on_uncaught_toggle = + should_abort_on_uncaught_toggle_.Serialize(ctx, creator); + + size_t id = 0; +#define V(PropertyName, TypeName) \ + do { \ + Local field = PropertyName(); \ + if (!field.IsEmpty()) { \ + size_t index = creator->AddData(field); \ + info.persistent_templates.push_back({#PropertyName, id, index}); \ + } \ + id++; \ + } while (0); + ENVIRONMENT_STRONG_PERSISTENT_TEMPLATES(V) +#undef V + + id = 0; +#define V(PropertyName, TypeName) \ + do { \ + Local field = PropertyName(); \ + if (!field.IsEmpty()) { \ + size_t index = creator->AddData(ctx, field); \ + info.persistent_values.push_back({#PropertyName, id, index}); \ + } \ + id++; \ + } while (0); + ENVIRONMENT_STRONG_PERSISTENT_VALUES(V) +#undef V + + info.context = creator->AddData(ctx, context()); + return info; +} + +std::ostream& operator<<(std::ostream& output, + const std::vector& vec) { + output << "{\n"; + for (const auto& info : vec) { + output << " { \"" << info.name << "\", " << std::to_string(info.id) << ", " + << std::to_string(info.index) << " },\n"; + } + output << "}"; + return output; +} + +std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) { + output << "{\n" + << "// -- async_hooks begins --\n" + << i.async_hooks << ",\n" + << "// -- async_hooks begins --\n" + << i.tick_info << ", // tick_info\n" + << i.immediate_info << ", // immediate_info\n" + << "// -- performance_state begins --\n" + << i.performance_state << ",\n" + << "// -- performance_state ends --\n" + << i.stream_base_state << ", // stream_base_state\n" + << i.should_abort_on_uncaught_toggle + << ", // should_abort_on_uncaught_toggle\n" + << "// -- persistent_templates begins --\n" + << i.persistent_templates << ",\n" + << "// persistent_templates ends --\n" + << "// -- persistent_values begins --\n" + << i.persistent_values << ",\n" + << "// -- persistent_values ends --\n" + << i.context << ", // context\n" + << "}"; + return output; +} + +void Environment::DeserializeProperties(const EnvSerializeInfo* info) { + Local ctx = context(); + + async_hooks_.Deserialize(ctx); + immediate_info_.Deserialize(ctx); + tick_info_.Deserialize(ctx); + performance_state_->Deserialize(ctx); + stream_base_state_.Deserialize(ctx); + should_abort_on_uncaught_toggle_.Deserialize(ctx); + + if (enabled_debug_list_.enabled(DebugCategory::MKSNAPSHOT)) { + fprintf(stderr, "deserializing...\n"); + std::cerr << *info << "\n"; + } + + const std::vector& templates = info->persistent_templates; + size_t i = 0; // index to the array + size_t id = 0; +#define V(PropertyName, TypeName) \ + do { \ + if (templates.size() > i && id == templates[i].id) { \ + const PropInfo& d = templates[i]; \ + Debug(this, \ + DebugCategory::MKSNAPSHOT, \ + "deserializing %s template, %" PRIu64 "/%" PRIu64 " (#%" PRIu64 \ + ") \n", \ + d.name.c_str(), \ + static_cast(i), \ + static_cast(templates.size()), \ + static_cast(id)); \ + DCHECK_EQ(d.name, #PropertyName); \ + MaybeLocal field = \ + isolate_->GetDataFromSnapshotOnce(d.index); \ + if (field.IsEmpty()) { \ + fprintf(stderr, \ + "Failed to deserialize environment template " #PropertyName \ + "\n"); \ + } \ + set_##PropertyName(field.ToLocalChecked()); \ + i++; \ + } \ + } while (0); \ + id++; + ENVIRONMENT_STRONG_PERSISTENT_TEMPLATES(V); +#undef V + + i = 0; // index to the array + id = 0; + const std::vector& values = info->persistent_values; +#define V(PropertyName, TypeName) \ + do { \ + if (values.size() > i && id == values[i].id) { \ + const PropInfo& d = values[i]; \ + Debug(this, \ + DebugCategory::MKSNAPSHOT, \ + "deserializing %s value, %" PRIu64 "/%" PRIu64 " (#%" PRIu64 \ + ") \n", \ + d.name.c_str(), \ + static_cast(i), \ + static_cast(values.size()), \ + static_cast(id)); \ + DCHECK_EQ(values[i].name, #PropertyName); \ + MaybeLocal field = \ + ctx->GetDataFromSnapshotOnce(values[i++].index); \ + if (field.IsEmpty()) { \ + fprintf(stderr, \ + "Failed to deserialize environment value " #PropertyName \ + "\n"); \ + } \ + set_##PropertyName(field.ToLocalChecked()); \ + } \ + } while (0); \ + id++; + ENVIRONMENT_STRONG_PERSISTENT_VALUES(V); +#undef V + + MaybeLocal ctx_from_snapshot = + ctx->GetDataFromSnapshotOnce(info->context); + if (ctx_from_snapshot.IsEmpty()) { + fprintf(stderr, + "Failed to deserialize context back reference from the snapshot\n"); + } + CHECK_EQ(ctx_from_snapshot.ToLocalChecked(), ctx); +} + void Environment::BuildEmbedderGraph(Isolate* isolate, EmbedderGraph* graph, void* data) { diff --git a/src/env.h b/src/env.h index 8c1bf71ddfaee4..c6f6ac481544f5 100644 --- a/src/env.h +++ b/src/env.h @@ -36,6 +36,7 @@ #include "node_binding.h" #include "node_main_instance.h" #include "node_options.h" +#include "node_perf_common.h" #include "req_wrap.h" #include "util.h" #include "uv.h" @@ -45,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -532,6 +534,7 @@ constexpr size_t kFsStatsBufferLength = class Environment; struct AllocatedBuffer; +typedef size_t SnapshotIndex; class IsolateData : public MemoryRetainer { public: IsolateData(v8::Isolate* isolate, @@ -645,6 +648,8 @@ namespace per_process { extern std::shared_ptr system_environment; } +struct EnvSerializeInfo; + class AsyncHooks : public MemoryRetainer { public: SET_MEMORY_INFO_NAME(AsyncHooks) @@ -722,9 +727,22 @@ class AsyncHooks : public MemoryRetainer { double old_default_trigger_async_id_; }; + struct SerializeInfo { + AliasedBufferInfo async_ids_stack; + AliasedBufferInfo fields; + AliasedBufferInfo async_id_fields; + SnapshotIndex js_execution_async_resources; + std::vector native_execution_async_resources; + }; + + SerializeInfo Serialize(v8::Local context, + v8::SnapshotCreator* creator); + void Deserialize(v8::Local context); + private: friend class Environment; // So we can call the constructor. - inline AsyncHooks(); + explicit AsyncHooks(v8::Isolate* isolate, const SerializeInfo* info); + // Stores the ids of the current execution context stack. AliasedFloat64Array async_ids_stack_; // Attached to a Uint32Array that tracks the number of active hooks for @@ -737,6 +755,9 @@ class AsyncHooks : public MemoryRetainer { v8::Global js_execution_async_resources_; std::vector> native_execution_async_resources_; + + // Non-empty during deserialization + const SerializeInfo* info_ = nullptr; }; class ImmediateInfo : public MemoryRetainer { @@ -758,9 +779,16 @@ class ImmediateInfo : public MemoryRetainer { SET_SELF_SIZE(ImmediateInfo) void MemoryInfo(MemoryTracker* tracker) const override; + struct SerializeInfo { + AliasedBufferInfo fields; + }; + SerializeInfo Serialize(v8::Local context, + v8::SnapshotCreator* creator); + void Deserialize(v8::Local context); + private: friend class Environment; // So we can call the constructor. - inline explicit ImmediateInfo(v8::Isolate* isolate); + explicit ImmediateInfo(v8::Isolate* isolate, const SerializeInfo* info); enum Fields { kCount, kRefCount, kHasOutstanding, kFieldsCount }; @@ -783,9 +811,16 @@ class TickInfo : public MemoryRetainer { TickInfo& operator=(TickInfo&&) = delete; ~TickInfo() = default; + struct SerializeInfo { + AliasedBufferInfo fields; + }; + SerializeInfo Serialize(v8::Local context, + v8::SnapshotCreator* creator); + void Deserialize(v8::Local context); + private: friend class Environment; // So we can call the constructor. - inline explicit TickInfo(v8::Isolate* isolate); + explicit TickInfo(v8::Isolate* isolate, const SerializeInfo* info); enum Fields { kHasTickScheduled = 0, kHasRejectionToWarn, kFieldsCount }; @@ -857,6 +892,27 @@ class CleanupHookCallback { uint64_t insertion_order_counter_; }; +struct PropInfo { + std::string name; // name for debugging + size_t id; // In the list - in case there are any empty entires + SnapshotIndex index; // In the snapshot +}; + +struct EnvSerializeInfo { + AsyncHooks::SerializeInfo async_hooks; + TickInfo::SerializeInfo tick_info; + ImmediateInfo::SerializeInfo immediate_info; + performance::PerformanceState::SerializeInfo performance_state; + AliasedBufferInfo stream_base_state; + AliasedBufferInfo should_abort_on_uncaught_toggle; + + std::vector persistent_templates; + std::vector persistent_values; + + SnapshotIndex context; + friend std::ostream& operator<<(std::ostream& o, const EnvSerializeInfo& i); +}; + class Environment : public MemoryRetainer { public: Environment(const Environment&) = delete; @@ -870,7 +926,13 @@ class Environment : public MemoryRetainer { bool IsRootNode() const override { return true; } void MemoryInfo(MemoryTracker* tracker) const override; + EnvSerializeInfo Serialize(v8::SnapshotCreator* creator); void CreateProperties(); + void DeserializeProperties(const EnvSerializeInfo* info); + + typedef void (*BaseObjectIterator)(size_t, BaseObject*); + void ForEachBaseObject(BaseObjectIterator iterator); + void PrintAllBaseObjects(); // Should be called before InitializeInspector() void InitializeDiagnostics(); #if HAVE_INSPECTOR @@ -925,14 +987,17 @@ class Environment : public MemoryRetainer { v8::Isolate* isolate, const std::vector& args, const std::vector& exec_args, + const EnvSerializeInfo* env_info, EnvironmentFlags::Flags flags, ThreadId thread_id); - void InitializeMainContext(v8::Local context); + void InitializeMainContext(v8::Local context, + const EnvSerializeInfo* env_info); // Create an Environment and initialize the provided main context for it. Environment(IsolateData* isolate_data, v8::Local context, const std::vector& args, const std::vector& exec_args, + const EnvSerializeInfo* env_info, EnvironmentFlags::Flags flags, ThreadId thread_id); ~Environment() override; @@ -1354,6 +1419,7 @@ class Environment : public MemoryRetainer { AliasedInt32Array stream_base_state_; + uint64_t environment_start_time_; std::unique_ptr performance_state_; std::unordered_map performance_marks_; diff --git a/src/node.cc b/src/node.cc index e27edec0efe7d3..e5658e546014a3 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1061,19 +1061,15 @@ int Start(int argc, char** argv) { Isolate::CreateParams params; const std::vector* indexes = nullptr; std::vector external_references; - + const EnvSerializeInfo* env_info = nullptr; bool force_no_snapshot = per_process::cli_options->per_isolate->no_node_snapshot; if (!force_no_snapshot) { v8::StartupData* blob = NodeMainInstance::GetEmbeddedSnapshotBlob(); if (blob != nullptr) { - // TODO(joyeecheung): collect external references and set it in - // params.external_references. - external_references = NodeMainInstance::CollectExternalReferences(); - external_references.push_back(reinterpret_cast(nullptr)); - params.external_references = external_references.data(); params.snapshot_blob = blob; indexes = NodeMainInstance::GetIsolateDataIndexes(); + env_info = NodeMainInstance::GetEnvSerializeInfo(); } } @@ -1083,7 +1079,7 @@ int Start(int argc, char** argv) { result.args, result.exec_args, indexes); - result.exit_code = main_instance.Run(); + result.exit_code = main_instance.Run(env_info); } TearDownOncePerProcess(); diff --git a/src/node_external_reference.h b/src/node_external_reference.h index 544236e3af669c..29dd4cb91ed663 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -46,7 +46,10 @@ class ExternalReferenceRegistry { std::vector external_references_; }; -#define EXTERNAL_REFERENCE_BINDING_LIST(V) +#define EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) V(process_object) + +#define EXTERNAL_REFERENCE_BINDING_LIST(V) \ + EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) } // namespace node diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 19f25574acbe95..9fefc371d66ab4 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -1,9 +1,9 @@ +#include "node_main_instance.h" +#include #include - -#include "node_errors.h" +#include "debug_utils-inl.h" #include "node_external_reference.h" #include "node_internals.h" -#include "node_main_instance.h" #include "node_options-inl.h" #include "node_v8_platform-inl.h" #include "util-inl.h" @@ -22,6 +22,7 @@ using v8::HandleScope; using v8::Isolate; using v8::Local; using v8::Locker; +using v8::Object; using v8::SealHandleScope; std::unique_ptr NodeMainInstance::registry_ = @@ -49,8 +50,6 @@ const std::vector& NodeMainInstance::CollectExternalReferences() { // Cannot be called more than once. CHECK_NULL(registry_); registry_.reset(new ExternalReferenceRegistry()); - - // TODO(joyeecheung): collect more external references here. return registry_->external_references(); } @@ -79,6 +78,15 @@ NodeMainInstance::NodeMainInstance( isolate_data_(nullptr), owns_isolate_(true) { params->array_buffer_allocator = array_buffer_allocator_.get(); + deserialize_mode_ = per_isolate_data_indexes != nullptr; + if (deserialize_mode_) { + // TODO(joyeecheung): collect external references and set it in + // params.external_references. + const std::vector& external_references = + CollectExternalReferences(); + params->external_references = external_references.data(); + } + isolate_ = Isolate::Allocate(); CHECK_NOT_NULL(isolate_); // Register the isolate on the platform before the isolate gets initialized, @@ -87,7 +95,6 @@ NodeMainInstance::NodeMainInstance( SetIsolateCreateParamsForNode(params); Isolate::Initialize(isolate_, *params); - deserialize_mode_ = per_isolate_data_indexes != nullptr; // If the indexes are not nullptr, we are not deserializing CHECK_IMPLIES(deserialize_mode_, params->external_references != nullptr); isolate_data_ = std::make_unique(isolate_, @@ -117,56 +124,57 @@ NodeMainInstance::~NodeMainInstance() { isolate_->Dispose(); } -int NodeMainInstance::Run() { +int NodeMainInstance::Run(const EnvSerializeInfo* env_info) { Locker locker(isolate_); Isolate::Scope isolate_scope(isolate_); HandleScope handle_scope(isolate_); int exit_code = 0; DeleteFnPtr env = - CreateMainEnvironment(&exit_code); + CreateMainEnvironment(&exit_code, env_info); CHECK_NOT_NULL(env); - Context::Scope context_scope(env->context()); + { + Context::Scope context_scope(env->context()); - if (exit_code == 0) { - LoadEnvironment(env.get()); + if (exit_code == 0) { + LoadEnvironment(env.get()); - env->set_trace_sync_io(env->options()->trace_sync_io); + env->set_trace_sync_io(env->options()->trace_sync_io); - { - SealHandleScope seal(isolate_); - bool more; - env->performance_state()->Mark( - node::performance::NODE_PERFORMANCE_MILESTONE_LOOP_START); - do { - uv_run(env->event_loop(), UV_RUN_DEFAULT); + { + SealHandleScope seal(isolate_); + bool more; + env->performance_state()->Mark( + node::performance::NODE_PERFORMANCE_MILESTONE_LOOP_START); + do { + uv_run(env->event_loop(), UV_RUN_DEFAULT); - per_process::v8_platform.DrainVMTasks(isolate_); + per_process::v8_platform.DrainVMTasks(isolate_); - more = uv_loop_alive(env->event_loop()); - if (more && !env->is_stopping()) continue; + more = uv_loop_alive(env->event_loop()); + if (more && !env->is_stopping()) continue; - if (!uv_loop_alive(env->event_loop())) { - EmitBeforeExit(env.get()); - } + if (!uv_loop_alive(env->event_loop())) { + EmitBeforeExit(env.get()); + } - // Emit `beforeExit` if the loop became alive either after emitting - // event, or after running some callbacks. - more = uv_loop_alive(env->event_loop()); - } while (more == true && !env->is_stopping()); - env->performance_state()->Mark( - node::performance::NODE_PERFORMANCE_MILESTONE_LOOP_EXIT); - } + // Emit `beforeExit` if the loop became alive either after emitting + // event, or after running some callbacks. + more = uv_loop_alive(env->event_loop()); + } while (more == true && !env->is_stopping()); + env->performance_state()->Mark( + node::performance::NODE_PERFORMANCE_MILESTONE_LOOP_EXIT); + } - env->set_trace_sync_io(false); - exit_code = EmitExit(env.get()); - } + env->set_trace_sync_io(false); + exit_code = EmitExit(env.get()); + } - ResetStdio(); + ResetStdio(); - // TODO(addaleax): Neither NODE_SHARED_MODE nor HAVE_INSPECTOR really - // make sense here. + // TODO(addaleax): Neither NODE_SHARED_MODE nor HAVE_INSPECTOR really + // make sense here. #if HAVE_INSPECTOR && defined(__POSIX__) && !defined(NODE_SHARED_MODE) struct sigaction act; memset(&act, 0, sizeof(act)); @@ -181,12 +189,26 @@ int NodeMainInstance::Run() { #if defined(LEAK_SANITIZER) __lsan_do_leak_check(); #endif + } return exit_code; } +void DeserializeNodeInternalFields(Local holder, + int index, + v8::StartupData payload, + void* env) { + if (payload.raw_size == 0) { + holder->SetAlignedPointerInInternalField(index, nullptr); + return; + } + // No embedder object in the builtin snapshot yet. + UNREACHABLE(); +} + DeleteFnPtr -NodeMainInstance::CreateMainEnvironment(int* exit_code) { +NodeMainInstance::CreateMainEnvironment(int* exit_code, + const EnvSerializeInfo* env_info) { *exit_code = 0; // Reset the exit code to 0 HandleScope handle_scope(isolate_); @@ -197,32 +219,48 @@ NodeMainInstance::CreateMainEnvironment(int* exit_code) { isolate_->GetHeapProfiler()->StartTrackingHeapObjects(true); } + CHECK_IMPLIES(deserialize_mode_, env_info != nullptr); Local context; + DeleteFnPtr env; + if (deserialize_mode_) { - context = - Context::FromSnapshot(isolate_, kNodeContextIndex).ToLocalChecked(); + env.reset(new Environment(isolate_data_.get(), + isolate_, + args_, + exec_args_, + env_info, + EnvironmentFlags::kDefaultFlags, + {})); + context = Context::FromSnapshot(isolate_, + kNodeContextIndex, + {DeserializeNodeInternalFields, env.get()}) + .ToLocalChecked(); + InitializeContextRuntime(context); SetIsolateErrorHandlers(isolate_, {}); } else { context = NewContext(isolate_); + Context::Scope context_scope(context); + env.reset(new Environment(isolate_data_.get(), + context, + args_, + exec_args_, + nullptr, + EnvironmentFlags::kDefaultFlags, + {})); } CHECK(!context.IsEmpty()); Context::Scope context_scope(context); - DeleteFnPtr env { CreateEnvironment( - isolate_data_.get(), - context, - args_, - exec_args_, - EnvironmentFlags::kDefaultFlags) }; + env->InitializeMainContext(context, env_info); - if (*exit_code != 0) { - return env; - } +#if HAVE_INSPECTOR + env->InitializeInspector({}); +#endif - if (env == nullptr) { - *exit_code = 1; + if (env->RunBootstrapping().IsEmpty()) { + return nullptr; } return env; diff --git a/src/node_main_instance.h b/src/node_main_instance.h index de1f4dca2b7cc6..6e38e95c26635c 100644 --- a/src/node_main_instance.h +++ b/src/node_main_instance.h @@ -14,6 +14,7 @@ namespace node { class ExternalReferenceRegistry; +struct EnvSerializeInfo; // TODO(joyeecheung): align this with the Worker/WorkerThreadData class. // We may be able to create an abstract class to reuse some of the routines. @@ -57,17 +58,18 @@ class NodeMainInstance { ~NodeMainInstance(); // Start running the Node.js instances, return the exit code when finished. - int Run(); + int Run(const EnvSerializeInfo* env_info); IsolateData* isolate_data() { return isolate_data_.get(); } DeleteFnPtr CreateMainEnvironment( - int* exit_code); + int* exit_code, const EnvSerializeInfo* env_info); // If nullptr is returned, the binary is not built with embedded // snapshot. static const std::vector* GetIsolateDataIndexes(); static v8::StartupData* GetEmbeddedSnapshotBlob(); + static const EnvSerializeInfo* GetEnvSerializeInfo(); static const std::vector& CollectExternalReferences(); static const size_t kNodeContextIndex = 0; diff --git a/src/node_perf.cc b/src/node_perf.cc index b5efc05689f20c..d71afc2d81c258 100644 --- a/src/node_perf.cc +++ b/src/node_perf.cc @@ -40,6 +40,52 @@ const uint64_t timeOrigin = PERFORMANCE_NOW(); const double timeOriginTimestamp = GetCurrentTimeInMicroseconds(); uint64_t performance_v8_start; +PerformanceState::PerformanceState(Isolate* isolate, + const PerformanceState::SerializeInfo* info) + : root(isolate, + sizeof(performance_state_internal), + MAYBE_FIELD_PTR(info, root)), + milestones(isolate, + offsetof(performance_state_internal, milestones), + NODE_PERFORMANCE_MILESTONE_INVALID, + root, + MAYBE_FIELD_PTR(info, milestones)), + observers(isolate, + offsetof(performance_state_internal, observers), + NODE_PERFORMANCE_ENTRY_TYPE_INVALID, + root, + MAYBE_FIELD_PTR(info, observers)) { + if (info == nullptr) { + for (size_t i = 0; i < milestones.Length(); i++) milestones[i] = -1.; + } +} + +PerformanceState::SerializeInfo PerformanceState::Serialize( + v8::Local context, v8::SnapshotCreator* creator) { + SerializeInfo info{root.Serialize(context, creator), + milestones.Serialize(context, creator), + observers.Serialize(context, creator)}; + return info; +} + +void PerformanceState::Deserialize(v8::Local context) { + root.Deserialize(context); + // This is just done to set up the pointers, we will actually reset + // all the milestones after deserialization. + milestones.Deserialize(context); + observers.Deserialize(context); +} + +std::ostream& operator<<(std::ostream& o, + const PerformanceState::SerializeInfo& i) { + o << "{\n" + << " " << i.root << ", // root\n" + << " " << i.milestones << ", // milestones\n" + << " " << i.observers << ", // observers\n" + << "}"; + return o; +} + void PerformanceState::Mark(enum PerformanceMilestone milestone, uint64_t ts) { this->milestones[milestone] = ts; @@ -111,8 +157,8 @@ void PerformanceEntry::Notify(Environment* env, Local object) { Context::Scope scope(env->context()); AliasedUint32Array& observers = env->performance_state()->observers; - if (type != NODE_PERFORMANCE_ENTRY_TYPE_INVALID && - observers[type]) { + if (!env->performance_entry_callback().IsEmpty() && + type != NODE_PERFORMANCE_ENTRY_TYPE_INVALID && observers[type]) { node::MakeCallback(env->isolate(), object.As(), env->performance_entry_callback(), diff --git a/src/node_perf_common.h b/src/node_perf_common.h index 75d266afc257e9..6c4c98813e71f0 100644 --- a/src/node_perf_common.h +++ b/src/node_perf_common.h @@ -8,6 +8,7 @@ #include "v8.h" #include +#include #include #include @@ -54,23 +55,17 @@ enum PerformanceEntryType { class PerformanceState { public: - explicit PerformanceState(v8::Isolate* isolate) : - root( - isolate, - sizeof(performance_state_internal)), - milestones( - isolate, - offsetof(performance_state_internal, milestones), - NODE_PERFORMANCE_MILESTONE_INVALID, - root), - observers( - isolate, - offsetof(performance_state_internal, observers), - NODE_PERFORMANCE_ENTRY_TYPE_INVALID, - root) { - for (size_t i = 0; i < milestones.Length(); i++) - milestones[i] = -1.; - } + struct SerializeInfo { + AliasedBufferInfo root; + AliasedBufferInfo milestones; + AliasedBufferInfo observers; + }; + + explicit PerformanceState(v8::Isolate* isolate, const SerializeInfo* info); + SerializeInfo Serialize(v8::Local context, + v8::SnapshotCreator* creator); + void Deserialize(v8::Local context); + friend std::ostream& operator<<(std::ostream& o, const SerializeInfo& i); AliasedUint8Array root; AliasedFloat64Array milestones; diff --git a/src/node_process_object.cc b/src/node_process_object.cc index 5bec65805a1024..4648cfbd11a925 100644 --- a/src/node_process_object.cc +++ b/src/node_process_object.cc @@ -1,7 +1,8 @@ #include "env-inl.h" +#include "node_external_reference.h" #include "node_internals.h" -#include "node_options-inl.h" #include "node_metadata.h" +#include "node_options-inl.h" #include "node_process.h" #include "node_revert.h" #include "util-inl.h" @@ -200,4 +201,11 @@ void PatchProcessObject(const FunctionCallbackInfo& args) { .FromJust()); } +void RegisterProcessExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(RawDebug); +} + } // namespace node + +NODE_MODULE_EXTERNAL_REFERENCE(process_object, + node::RegisterProcessExternalReferences) diff --git a/src/node_snapshot_stub.cc b/src/node_snapshot_stub.cc index fac03b0c87af5d..9d7b085994bf9f 100644 --- a/src/node_snapshot_stub.cc +++ b/src/node_snapshot_stub.cc @@ -14,4 +14,8 @@ const std::vector* NodeMainInstance::GetIsolateDataIndexes() { return nullptr; } +const EnvSerializeInfo* NodeMainInstance::GetEnvSerializeInfo() { + return nullptr; +} + } // namespace node diff --git a/src/util.h b/src/util.h index 8b8a63adca9f93..71f3886365e28a 100644 --- a/src/util.h +++ b/src/util.h @@ -805,6 +805,7 @@ std::unique_ptr static_unique_pointer_cast(std::unique_ptr&& ptr) { return std::unique_ptr(static_cast(ptr.release())); } +#define MAYBE_FIELD_PTR(ptr, field) ptr == nullptr ? nullptr : &(ptr->field) } // namespace node #endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS diff --git a/tools/snapshot/snapshot_builder.cc b/tools/snapshot/snapshot_builder.cc index 7d292391ab334a..7086a98a6c80c8 100644 --- a/tools/snapshot/snapshot_builder.cc +++ b/tools/snapshot/snapshot_builder.cc @@ -1,6 +1,7 @@ #include "snapshot_builder.h" #include #include +#include "debug_utils-inl.h" #include "env-inl.h" #include "node_external_reference.h" #include "node_internals.h" @@ -12,6 +13,8 @@ namespace node { using v8::Context; using v8::HandleScope; using v8::Isolate; +using v8::Local; +using v8::Object; using v8::SnapshotCreator; using v8::StartupData; @@ -23,10 +26,12 @@ void WriteVector(std::stringstream* ss, const T* vec, size_t size) { } std::string FormatBlob(v8::StartupData* blob, - const std::vector& isolate_data_indexes) { + const std::vector& isolate_data_indexes, + const EnvSerializeInfo& env_info) { std::stringstream ss; ss << R"(#include +#include "env.h" #include "node_main_instance.h" #include "v8.h" @@ -56,17 +61,39 @@ static const std::vector isolate_data_indexes { const std::vector* NodeMainInstance::GetIsolateDataIndexes() { return &isolate_data_indexes; } + +static const EnvSerializeInfo env_info )" + << env_info << R"(; + +const EnvSerializeInfo* NodeMainInstance::GetEnvSerializeInfo() { + return &env_info; +} + } // namespace node )"; return ss.str(); } +static v8::StartupData SerializeNodeContextInternalFields(Local holder, + int index, + void* env) { + void* ptr = holder->GetAlignedPointerFromInternalField(index); + if (ptr == nullptr || ptr == env) { + return StartupData{nullptr, 0}; + } + if (ptr == env && index == ContextEmbedderIndex::kEnvironment) { + return StartupData{nullptr, 0}; + } + + // No embedder objects in the builtin snapshot yet. + UNREACHABLE(); + return StartupData{nullptr, 0}; +} + std::string SnapshotBuilder::Generate( const std::vector args, const std::vector exec_args) { - const std::vector& external_references = - NodeMainInstance::CollectExternalReferences(); Isolate* isolate = Isolate::Allocate(); per_process::v8_platform.Platform()->RegisterIsolate(isolate, uv_default_loop()); @@ -75,7 +102,12 @@ std::string SnapshotBuilder::Generate( { std::vector isolate_data_indexes; + EnvSerializeInfo env_info; + + const std::vector& external_references = + NodeMainInstance::CollectExternalReferences(); SnapshotCreator creator(isolate, external_references.data()); + Environment* env; { main_instance = NodeMainInstance::Create(isolate, @@ -83,11 +115,28 @@ std::string SnapshotBuilder::Generate( per_process::v8_platform.Platform(), args, exec_args); + HandleScope scope(isolate); creator.SetDefaultContext(Context::New(isolate)); isolate_data_indexes = main_instance->isolate_data()->Serialize(&creator); - size_t index = creator.AddContext(NewContext(isolate)); + Local context = NewContext(isolate); + Context::Scope context_scope(context); + + env = new Environment(main_instance->isolate_data(), + context, + args, + exec_args, + nullptr, + node::EnvironmentFlags::kDefaultFlags, + {}); + if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { + env->PrintAllBaseObjects(); + printf("Environment = %p\n", env); + } + env_info = env->Serialize(&creator); + size_t index = creator.AddContext( + context, {SerializeNodeContextInternalFields, env}); CHECK_EQ(index, NodeMainInstance::kNodeContextIndex); } @@ -97,8 +146,9 @@ std::string SnapshotBuilder::Generate( CHECK(blob.CanBeRehashed()); // Must be done while the snapshot creator isolate is entered i.e. the // creator is still alive. + FreeEnvironment(env); main_instance->Dispose(); - result = FormatBlob(&blob, isolate_data_indexes); + result = FormatBlob(&blob, isolate_data_indexes, env_info); delete[] blob.data; } From aa3fabebf5707f9b9ffc530686505b0f7ce4e151 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 22 Apr 2020 04:10:10 +0800 Subject: [PATCH 05/13] src: make code cache test work with snapshots Keep track of snapshotted modules in JS land, and move bootstrap switches into StartExecution() so that they are not included into part of the environment-independent bootstrap process. --- lib/internal/bootstrap/node.js | 32 +++++++++++++++--------- src/env.cc | 42 +++++++++++++++++--------------- src/env.h | 4 +++ src/node.cc | 1 + src/node_config.cc | 3 --- src/node_native_module_env.cc | 14 +++++++++++ test/parallel/test-code-cache.js | 14 ++++++++--- 7 files changed, 72 insertions(+), 38 deletions(-) diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js index 92dc2a6cc7d2d6..b39bfe078f450d 100644 --- a/lib/internal/bootstrap/node.js +++ b/lib/internal/bootstrap/node.js @@ -57,7 +57,8 @@ process.domain = null; process._exiting = false; // process.config is serialized config.gypi -process.config = JSONParse(internalBinding('native_module').config); +const nativeModule = internalBinding('native_module'); +process.config = JSONParse(nativeModule.config); require('internal/worker/js_transferable').setup(); // Bootstrappers for all threads, including worker threads and main thread @@ -192,21 +193,28 @@ process.assert = deprecate( // TODO(joyeecheung): this property has not been well-maintained, should we // deprecate it in favor of a better API? const { isDebugBuild, hasOpenSSL, hasInspector } = config; +const features = { + inspector: hasInspector, + debug: isDebugBuild, + uv: true, + ipv6: true, // TODO(bnoordhuis) ping libuv + tls_alpn: hasOpenSSL, + tls_sni: hasOpenSSL, + tls_ocsp: hasOpenSSL, + tls: hasOpenSSL, + // This needs to be dynamic because snapshot is built without code cache. + // TODO(joyeecheung): https://github.com/nodejs/node/issues/31074 + // Make it possible to build snapshot with code cache + get cached_builtins() { + return nativeModule.hasCachedBuiltins(); + } +}; + ObjectDefineProperty(process, 'features', { enumerable: true, writable: false, configurable: false, - value: { - inspector: hasInspector, - debug: isDebugBuild, - uv: true, - ipv6: true, // TODO(bnoordhuis) ping libuv - tls_alpn: hasOpenSSL, - tls_sni: hasOpenSSL, - tls_ocsp: hasOpenSSL, - tls: hasOpenSSL, - cached_builtins: config.hasCachedBuiltins, - } + value: features }); { diff --git a/src/env.cc b/src/env.cc index dde1c16db12725..0f9c1a326740b6 100644 --- a/src/env.cc +++ b/src/env.cc @@ -1208,6 +1208,11 @@ EnvSerializeInfo Environment::Serialize(SnapshotCreator* creator) { EnvSerializeInfo info; Local ctx = context(); + // Currently all modules are compiled without cache in builtin snapshot + // builder. + info.native_modules = std::vector( + native_modules_without_cache.begin(), native_modules_without_cache.end()); + info.async_hooks = async_hooks_.Serialize(ctx, creator); info.immediate_info = immediate_info_.Serialize(ctx, creator); info.tick_info = tick_info_.Serialize(ctx, creator); @@ -1257,11 +1262,24 @@ std::ostream& operator<<(std::ostream& output, return output; } +std::ostream& operator<<(std::ostream& output, + const std::vector& vec) { + output << "{\n"; + for (const auto& info : vec) { + output << " \"" << info << "\",\n"; + } + output << "}"; + return output; +} + std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) { output << "{\n" + << "// -- native_modules begins --\n" + << i.native_modules << ",\n" + << "// -- native_modules ends --\n" << "// -- async_hooks begins --\n" << i.async_hooks << ",\n" - << "// -- async_hooks begins --\n" + << "// -- async_hooks ends --\n" << i.tick_info << ", // tick_info\n" << i.immediate_info << ", // immediate_info\n" << "// -- performance_state begins --\n" @@ -1284,6 +1302,7 @@ std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) { void Environment::DeserializeProperties(const EnvSerializeInfo* info) { Local ctx = context(); + native_modules_in_snapshot = info->native_modules; async_hooks_.Deserialize(ctx); immediate_info_.Deserialize(ctx); tick_info_.Deserialize(ctx); @@ -1303,14 +1322,6 @@ void Environment::DeserializeProperties(const EnvSerializeInfo* info) { do { \ if (templates.size() > i && id == templates[i].id) { \ const PropInfo& d = templates[i]; \ - Debug(this, \ - DebugCategory::MKSNAPSHOT, \ - "deserializing %s template, %" PRIu64 "/%" PRIu64 " (#%" PRIu64 \ - ") \n", \ - d.name.c_str(), \ - static_cast(i), \ - static_cast(templates.size()), \ - static_cast(id)); \ DCHECK_EQ(d.name, #PropertyName); \ MaybeLocal field = \ isolate_->GetDataFromSnapshotOnce(d.index); \ @@ -1334,23 +1345,16 @@ void Environment::DeserializeProperties(const EnvSerializeInfo* info) { do { \ if (values.size() > i && id == values[i].id) { \ const PropInfo& d = values[i]; \ - Debug(this, \ - DebugCategory::MKSNAPSHOT, \ - "deserializing %s value, %" PRIu64 "/%" PRIu64 " (#%" PRIu64 \ - ") \n", \ - d.name.c_str(), \ - static_cast(i), \ - static_cast(values.size()), \ - static_cast(id)); \ - DCHECK_EQ(values[i].name, #PropertyName); \ + DCHECK_EQ(d.name, #PropertyName); \ MaybeLocal field = \ - ctx->GetDataFromSnapshotOnce(values[i++].index); \ + ctx->GetDataFromSnapshotOnce(d.index); \ if (field.IsEmpty()) { \ fprintf(stderr, \ "Failed to deserialize environment value " #PropertyName \ "\n"); \ } \ set_##PropertyName(field.ToLocalChecked()); \ + i++; \ } \ } while (0); \ id++; diff --git a/src/env.h b/src/env.h index c6f6ac481544f5..d475fc78fb9484 100644 --- a/src/env.h +++ b/src/env.h @@ -899,6 +899,7 @@ struct PropInfo { }; struct EnvSerializeInfo { + std::vector native_modules; AsyncHooks::SerializeInfo async_hooks; TickInfo::SerializeInfo tick_info; ImmediateInfo::SerializeInfo immediate_info; @@ -1086,6 +1087,9 @@ class Environment : public MemoryRetainer { std::set native_modules_with_cache; std::set native_modules_without_cache; + // This is only filled during deserialization. We use a vector since + // it's only used for tests. + std::vector native_modules_in_snapshot; std::unordered_multimap hash_to_module_map; std::unordered_map id_to_module_map; diff --git a/src/node.cc b/src/node.cc index e5658e546014a3..c7da03616e65f9 100644 --- a/src/node.cc +++ b/src/node.cc @@ -350,6 +350,7 @@ MaybeLocal Environment::BootstrapNode() { return scope.EscapeMaybe(result); } + // TODO(joyeecheung): skip these in the snapshot building for workers. auto thread_switch_id = is_main_thread() ? "internal/bootstrap/switches/is_main_thread" : "internal/bootstrap/switches/is_not_main_thread"; diff --git a/src/node_config.cc b/src/node_config.cc index 6ee3164a134fe8..2d8ad25bbe9c02 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -84,9 +84,6 @@ static void Initialize(Local target, #if defined HAVE_DTRACE || defined HAVE_ETW READONLY_TRUE_PROPERTY(target, "hasDtrace"); #endif - - READONLY_PROPERTY(target, "hasCachedBuiltins", - v8::Boolean::New(isolate, native_module::has_code_cache)); } // InitConfig } // namespace node diff --git a/src/node_native_module_env.cc b/src/node_native_module_env.cc index a3a42a3ec4b9a6..e38529eefae9c4 100644 --- a/src/node_native_module_env.cc +++ b/src/node_native_module_env.cc @@ -94,6 +94,14 @@ void NativeModuleEnv::GetCacheUsage(const FunctionCallbackInfo& args) { OneByteString(isolate, "compiledWithoutCache"), ToJsSet(context, env->native_modules_without_cache)) .FromJust(); + + result + ->Set(env->context(), + OneByteString(isolate, "compiledInSnapshot"), + ToV8Value(env->context(), env->native_modules_in_snapshot) + .ToLocalChecked()) + .FromJust(); + args.GetReturnValue().Set(result); } @@ -155,6 +163,11 @@ MaybeLocal NativeModuleEnv::LookupAndCompile( return maybe; } +void HasCachedBuiltins(const FunctionCallbackInfo& args) { + args.GetReturnValue().Set( + v8::Boolean::New(args.GetIsolate(), has_code_cache)); +} + // TODO(joyeecheung): It is somewhat confusing that Class::Initialize // is used to initialize to the binding, but it is the current convention. // Rename this across the code base to something that makes more sense. @@ -198,6 +211,7 @@ void NativeModuleEnv::Initialize(Local target, env->SetMethod(target, "getCacheUsage", NativeModuleEnv::GetCacheUsage); env->SetMethod(target, "compileFunction", NativeModuleEnv::CompileFunction); + env->SetMethod(target, "hasCachedBuiltins", HasCachedBuiltins); // internalBinding('native_module') should be frozen target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust(); } diff --git a/test/parallel/test-code-cache.js b/test/parallel/test-code-cache.js index 3c4488c557d524..1b151e269dcfaf 100644 --- a/test/parallel/test-code-cache.js +++ b/test/parallel/test-code-cache.js @@ -22,12 +22,16 @@ for (const key of canBeRequired) { // The computation has to be delayed until we have done loading modules const { compiledWithoutCache, - compiledWithCache + compiledWithCache, + compiledInSnapshot } = getCacheUsage(); -const loadedModules = process.moduleLoadList - .filter((m) => m.startsWith('NativeModule')) +function extractModules(list) { + return list.filter((m) => m.startsWith('NativeModule')) .map((m) => m.replace('NativeModule ', '')); +} + +const loadedModules = extractModules(process.moduleLoadList); // Cross-compiled binaries do not have code cache, verifies that the builtins // are all compiled without cache and we are doing the bookkeeping right. @@ -62,7 +66,9 @@ if (!process.features.cached_builtins) { if (cannotBeRequired.has(key) && !compiledWithoutCache.has(key)) { wrong.push(`"${key}" should've been compiled **without** code cache`); } - if (canBeRequired.has(key) && !compiledWithCache.has(key)) { + if (canBeRequired.has(key) && + !compiledWithCache.has(key) && + compiledInSnapshot.indexOf(key) === -1) { wrong.push(`"${key}" should've been compiled **with** code cache`); } } From 7fb9589b4483292d5716524cc4ac0eee95372d15 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 21 Apr 2020 07:32:31 +0800 Subject: [PATCH 06/13] src: snapshot loaders This runs `lib/internal/bootstrap/loaders.js` before creating the builtin snapshot and deserialize the loaders from the snapshot in deserialization mode. --- src/node_binding.cc | 11 ++++++++++- src/node_external_reference.h | 5 ++++- src/node_main_instance.cc | 9 ++++++++- src/node_native_module_env.cc | 15 +++++++++++++++ src/node_native_module_env.h | 2 ++ tools/snapshot/snapshot_builder.cc | 1 + 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/node_binding.cc b/src/node_binding.cc index 80ab4e69e110d0..4dbf56b99a3ddb 100644 --- a/src/node_binding.cc +++ b/src/node_binding.cc @@ -1,7 +1,8 @@ #include "node_binding.h" -#include "node_errors.h" #include #include "env-inl.h" +#include "node_errors.h" +#include "node_external_reference.h" #include "node_native_module_env.h" #include "util.h" @@ -676,5 +677,13 @@ void RegisterBuiltinModules() { #undef V } +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(GetLinkedBinding); + registry->Register(GetInternalBinding); +} + } // namespace binding } // namespace node + +NODE_MODULE_EXTERNAL_REFERENCE(binding, + node::binding::RegisterExternalReferences) diff --git a/src/node_external_reference.h b/src/node_external_reference.h index 29dd4cb91ed663..8c97eca2264a60 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -46,7 +46,10 @@ class ExternalReferenceRegistry { std::vector external_references_; }; -#define EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) V(process_object) +#define EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) \ + V(binding) \ + V(native_module) \ + V(process_object) #define EXTERNAL_REFERENCE_BINDING_LIST(V) \ EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 9fefc371d66ab4..617a71e7cac60d 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -259,10 +259,17 @@ NodeMainInstance::CreateMainEnvironment(int* exit_code, env->InitializeInspector({}); #endif - if (env->RunBootstrapping().IsEmpty()) { + if (!deserialize_mode_ && env->RunBootstrapping().IsEmpty()) { return nullptr; } + if (deserialize_mode_ && env->BootstrapNode().IsEmpty()) { + return nullptr; + } + + CHECK(env->req_wrap_queue()->IsEmpty()); + CHECK(env->handle_wrap_queue()->IsEmpty()); + env->set_has_run_bootstrapping_code(true); return env; } diff --git a/src/node_native_module_env.cc b/src/node_native_module_env.cc index e38529eefae9c4..f388c696b0ad91 100644 --- a/src/node_native_module_env.cc +++ b/src/node_native_module_env.cc @@ -1,5 +1,6 @@ #include "node_native_module_env.h" #include "env-inl.h" +#include "node_external_reference.h" namespace node { namespace native_module { @@ -216,8 +217,22 @@ void NativeModuleEnv::Initialize(Local target, target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust(); } +void NativeModuleEnv::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(ConfigStringGetter); + registry->Register(ModuleIdsGetter); + registry->Register(ModuleIdsGetter); + registry->Register(GetModuleCategories); + registry->Register(GetCacheUsage); + registry->Register(CompileFunction); + registry->Register(HasCachedBuiltins); +} + } // namespace native_module } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL( native_module, node::native_module::NativeModuleEnv::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE( + native_module, + node::native_module::NativeModuleEnv::RegisterExternalReferences) diff --git a/src/node_native_module_env.h b/src/node_native_module_env.h index bc36be75109639..0a53771ff5d1ca 100644 --- a/src/node_native_module_env.h +++ b/src/node_native_module_env.h @@ -7,6 +7,7 @@ namespace node { class Environment; +class ExternalReferenceRegistry; namespace native_module { @@ -14,6 +15,7 @@ extern const bool has_code_cache; class NativeModuleEnv { public: + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); static void Initialize(v8::Local target, v8::Local unused, v8::Local context, diff --git a/tools/snapshot/snapshot_builder.cc b/tools/snapshot/snapshot_builder.cc index 7086a98a6c80c8..364202711105a4 100644 --- a/tools/snapshot/snapshot_builder.cc +++ b/tools/snapshot/snapshot_builder.cc @@ -130,6 +130,7 @@ std::string SnapshotBuilder::Generate( nullptr, node::EnvironmentFlags::kDefaultFlags, {}); + env->BootstrapInternalLoaders().ToLocalChecked(); if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { env->PrintAllBaseObjects(); printf("Environment = %p\n", env); From ea4446495e820ec8ea7b7a55ca892fecfaaa9614 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Thu, 7 May 2020 20:21:45 +0800 Subject: [PATCH 07/13] src: reset zero fill toggle at pre-execution The connection between the JS land zero fill toggle and the C++ one in the NodeArrayBufferAllocator gets lost if the toggle is deserialized from the snapshot, because V8 owns the underlying memory of this toggle. This resets the connection at pre-execution. --- lib/buffer.js | 21 ++-------- lib/internal/bootstrap/pre_execution.js | 6 +++ lib/internal/buffer.js | 27 ++++++++++++- src/node_buffer.cc | 51 ++++++++++++++----------- 4 files changed, 64 insertions(+), 41 deletions(-) diff --git a/lib/buffer.js b/lib/buffer.js index 90ebe8866b582c..4119dd55eaacb7 100644 --- a/lib/buffer.js +++ b/lib/buffer.js @@ -55,8 +55,7 @@ const { swap32: _swap32, swap64: _swap64, kMaxLength, - kStringMaxLength, - zeroFill: bindingZeroFill + kStringMaxLength } = internalBinding('buffer'); const { getOwnNonIndexProperties, @@ -102,7 +101,8 @@ const { const { FastBuffer, markAsUntransferable, - addBufferPrototypeMethods + addBufferPrototypeMethods, + createUnsafeBuffer } = require('internal/buffer'); const TypedArrayPrototype = ObjectGetPrototypeOf(Uint8ArrayPrototype); @@ -132,25 +132,10 @@ const constants = ObjectDefineProperties({}, { Buffer.poolSize = 8 * 1024; let poolSize, poolOffset, allocPool; -// A toggle used to access the zero fill setting of the array buffer allocator -// in C++. -// |zeroFill| can be undefined when running inside an isolate where we -// do not own the ArrayBuffer allocator. Zero fill is always on in that case. -const zeroFill = bindingZeroFill || [0]; - const encodingsMap = ObjectCreate(null); for (let i = 0; i < encodings.length; ++i) encodingsMap[encodings[i]] = i; -function createUnsafeBuffer(size) { - zeroFill[0] = 0; - try { - return new FastBuffer(size); - } finally { - zeroFill[0] = 1; - } -} - function createPool() { poolSize = Buffer.poolSize; allocPool = createUnsafeBuffer(poolSize).buffer; diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index f1e2bcfb9da643..b384349666d919 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -10,11 +10,17 @@ const { getOptionValue, shouldNotRegisterESMLoader } = require('internal/options'); +const { reconnectZeroFillToggle } = require('internal/buffer'); + const { Buffer } = require('buffer'); const { ERR_MANIFEST_ASSERT_INTEGRITY } = require('internal/errors').codes; const assert = require('internal/assert'); function prepareMainThreadExecution(expandArgv1 = false) { + // TODO(joyeecheung): this is also necessary for workers when they deserialize + // this toggle from the snapshot. + reconnectZeroFillToggle(); + // Patch the process object with legacy properties and normalizations patchProcessObject(expandArgv1); setupTraceCategoryState(); diff --git a/lib/internal/buffer.js b/lib/internal/buffer.js index be509088ada1b6..aed6b24f292354 100644 --- a/lib/internal/buffer.js +++ b/lib/internal/buffer.js @@ -25,7 +25,8 @@ const { latin1Write, hexWrite, ucs2Write, - utf8Write + utf8Write, + getZeroFillToggle } = internalBinding('buffer'); const { untransferable_object_private_symbol, @@ -1019,8 +1020,32 @@ function markAsUntransferable(obj) { setHiddenValue(obj, untransferable_object_private_symbol, true); } +// A toggle used to access the zero fill setting of the array buffer allocator +// in C++. +// |zeroFill| can be undefined when running inside an isolate where we +// do not own the ArrayBuffer allocator. Zero fill is always on in that case. +let zeroFill = getZeroFillToggle(); +function createUnsafeBuffer(size) { + zeroFill[0] = 0; + try { + return new FastBuffer(size); + } finally { + zeroFill[0] = 1; + } +} + +// The connection between the JS land zero fill toggle and the +// C++ one in the NodeArrayBufferAllocator gets lost if the toggle +// is deserialized from the snapshot, because V8 owns the underlying +// memory of this toggle. This resets the connection. +function reconnectZeroFillToggle() { + zeroFill = getZeroFillToggle(); +} + module.exports = { FastBuffer, addBufferPrototypeMethods, markAsUntransferable, + createUnsafeBuffer, + reconnectZeroFillToggle }; diff --git a/src/node_buffer.cc b/src/node_buffer.cc index 020b570cea3304..2028849775a98c 100644 --- a/src/node_buffer.cc +++ b/src/node_buffer.cc @@ -1117,6 +1117,34 @@ void SetBufferPrototype(const FunctionCallbackInfo& args) { env->set_buffer_prototype_object(proto); } +void GetZeroFillToggle(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + NodeArrayBufferAllocator* allocator = env->isolate_data()->node_allocator(); + Local ab; + // It can be a nullptr when running inside an isolate where we + // do not own the ArrayBuffer allocator. + if (allocator == nullptr) { + // Create a dummy Uint32Array - the JS land can only toggle the C++ land + // setting when the allocator uses our toggle. With this the toggle in JS + // land results in no-ops. + ab = ArrayBuffer::New(env->isolate(), sizeof(uint32_t)); + } else { + uint32_t* zero_fill_field = allocator->zero_fill_field(); + std::unique_ptr backing = + ArrayBuffer::NewBackingStore(zero_fill_field, + sizeof(*zero_fill_field), + [](void*, size_t, void*) {}, + nullptr); + ab = ArrayBuffer::New(env->isolate(), std::move(backing)); + } + + ab->SetPrivate( + env->context(), + env->untransferable_object_private_symbol(), + True(env->isolate())).Check(); + + args.GetReturnValue().Set(Uint32Array::New(ab, 0, 1)); +} void Initialize(Local target, Local unused, @@ -1165,28 +1193,7 @@ void Initialize(Local target, env->SetMethod(target, "ucs2Write", StringWrite); env->SetMethod(target, "utf8Write", StringWrite); - // It can be a nullptr when running inside an isolate where we - // do not own the ArrayBuffer allocator. - if (NodeArrayBufferAllocator* allocator = - env->isolate_data()->node_allocator()) { - uint32_t* zero_fill_field = allocator->zero_fill_field(); - std::unique_ptr backing = - ArrayBuffer::NewBackingStore(zero_fill_field, - sizeof(*zero_fill_field), - [](void*, size_t, void*){}, - nullptr); - Local array_buffer = - ArrayBuffer::New(env->isolate(), std::move(backing)); - array_buffer->SetPrivate( - env->context(), - env->untransferable_object_private_symbol(), - True(env->isolate())).Check(); - CHECK(target - ->Set(env->context(), - FIXED_ONE_BYTE_STRING(env->isolate(), "zeroFill"), - Uint32Array::New(array_buffer, 0, 1)) - .FromJust()); - } + env->SetMethod(target, "getZeroFillToggle", GetZeroFillToggle); } } // anonymous namespace From c7007ca59fa2a84527f09413bd0e9bbe93ee7a01 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 23 Jun 2020 03:31:52 +0800 Subject: [PATCH 08/13] bootstrap: build fast APIs in pre-execution Fast APIs need to work with ArrayBuffers which we need to rebuild connections to after deserializing them from the snapshot. For now, postpone their creation until pre-execution to simplify the process. --- lib/internal/bootstrap/node.js | 2 - lib/internal/bootstrap/pre_execution.js | 14 +++- lib/internal/process/per_thread.js | 87 ++++++++++++++----------- src/node_process_methods.cc | 17 +++-- test/cctest/test_base_object_ptr.cc | 2 +- 5 files changed, 71 insertions(+), 51 deletions(-) diff --git a/lib/internal/bootstrap/node.js b/lib/internal/bootstrap/node.js index b39bfe078f450d..551367583e55a8 100644 --- a/lib/internal/bootstrap/node.js +++ b/lib/internal/bootstrap/node.js @@ -80,8 +80,6 @@ const rawMethods = internalBinding('process_methods'); const wrapped = perThreadSetup.wrapProcessMethods(rawMethods); process._rawDebug = wrapped._rawDebug; - process.hrtime = wrapped.hrtime; - process.hrtime.bigint = wrapped.hrtimeBigInt; process.cpuUsage = wrapped.cpuUsage; process.resourceUsage = wrapped.resourceUsage; process.memoryUsage = wrapped.memoryUsage; diff --git a/lib/internal/bootstrap/pre_execution.js b/lib/internal/bootstrap/pre_execution.js index b384349666d919..7e24bd16425b16 100644 --- a/lib/internal/bootstrap/pre_execution.js +++ b/lib/internal/bootstrap/pre_execution.js @@ -74,11 +74,19 @@ function prepareMainThreadExecution(expandArgv1 = false) { } function patchProcessObject(expandArgv1) { + const binding = internalBinding('process_methods'); + binding.patchProcessObject(process); + + // TODO(joyeecheung): snapshot fast APIs (which need to work with + // array buffers, whose connection with C++ needs to be rebuilt after + // deserialization). const { - patchProcessObject: patchProcessObjectNative - } = internalBinding('process_methods'); + hrtime, + hrtimeBigInt + } = require('internal/process/per_thread').getFastAPIs(binding); - patchProcessObjectNative(process); + process.hrtime = hrtime; + process.hrtime.bigint = hrtimeBigInt; ObjectDefineProperty(process, 'argv0', { enumerable: true, diff --git a/lib/internal/process/per_thread.js b/lib/internal/process/per_thread.js index 303e71703d2fe5..33104dfb3db11c 100644 --- a/lib/internal/process/per_thread.js +++ b/lib/internal/process/per_thread.js @@ -37,10 +37,56 @@ function assert(x, msg) { if (!x) throw new ERR_ASSERTION(msg || 'assertion error'); } +function getFastAPIs(binding) { + const { + hrtime: _hrtime + } = binding.getFastAPIs(); + + // The 3 entries filled in by the original process.hrtime contains + // the upper/lower 32 bits of the second part of the value, + // and the remaining nanoseconds of the value. + const hrValues = new Uint32Array(_hrtime.buffer); + + function hrtime(time) { + _hrtime.hrtime(); + + if (time !== undefined) { + if (!ArrayIsArray(time)) { + throw new ERR_INVALID_ARG_TYPE('time', 'Array', time); + } + if (time.length !== 2) { + throw new ERR_OUT_OF_RANGE('time', 2, time.length); + } + + const sec = (hrValues[0] * 0x100000000 + hrValues[1]) - time[0]; + const nsec = hrValues[2] - time[1]; + const needsBorrow = nsec < 0; + return [needsBorrow ? sec - 1 : sec, needsBorrow ? nsec + 1e9 : nsec]; + } + + return [ + hrValues[0] * 0x100000000 + hrValues[1], + hrValues[2] + ]; + } + + // Use a BigUint64Array in the closure because this is actually a bit + // faster than simply returning a BigInt from C++ in V8 7.1. + const hrBigintValues = new BigUint64Array(_hrtime.buffer, 0, 1); + function hrtimeBigInt() { + _hrtime.hrtimeBigInt(); + return hrBigintValues[0]; + } + + return { + hrtime, + hrtimeBigInt, + }; +} + // The execution of this function itself should not cause any side effects. function wrapProcessMethods(binding) { const { - hrtime: _hrtime, cpuUsage: _cpuUsage, memoryUsage: _memoryUsage, resourceUsage: _resourceUsage @@ -109,42 +155,6 @@ function wrapProcessMethods(binding) { num >= 0; } - // The 3 entries filled in by the original process.hrtime contains - // the upper/lower 32 bits of the second part of the value, - // and the remaining nanoseconds of the value. - const hrValues = new Uint32Array(_hrtime.buffer); - - function hrtime(time) { - _hrtime.hrtime(); - - if (time !== undefined) { - if (!ArrayIsArray(time)) { - throw new ERR_INVALID_ARG_TYPE('time', 'Array', time); - } - if (time.length !== 2) { - throw new ERR_OUT_OF_RANGE('time', 2, time.length); - } - - const sec = (hrValues[0] * 0x100000000 + hrValues[1]) - time[0]; - const nsec = hrValues[2] - time[1]; - const needsBorrow = nsec < 0; - return [needsBorrow ? sec - 1 : sec, needsBorrow ? nsec + 1e9 : nsec]; - } - - return [ - hrValues[0] * 0x100000000 + hrValues[1], - hrValues[2] - ]; - } - - // Use a BigUint64Array in the closure because this is actually a bit - // faster than simply returning a BigInt from C++ in V8 7.1. - const hrBigintValues = new BigUint64Array(_hrtime.buffer, 0, 1); - function hrtimeBigInt() { - _hrtime.hrtimeBigInt(); - return hrBigintValues[0]; - } - const memValues = new Float64Array(5); function memoryUsage() { _memoryUsage(memValues); @@ -225,8 +235,6 @@ function wrapProcessMethods(binding) { return { _rawDebug, - hrtime, - hrtimeBigInt, cpuUsage, resourceUsage, memoryUsage, @@ -356,6 +364,7 @@ function toggleTraceCategoryState(asyncHooksEnabled) { module.exports = { toggleTraceCategoryState, + getFastAPIs, assert, buildAllowedFlags, wrapProcessMethods diff --git a/src/node_process_methods.cc b/src/node_process_methods.cc index aa186185988a0a..e714cdd30f99b1 100644 --- a/src/node_process_methods.cc +++ b/src/node_process_methods.cc @@ -504,6 +504,16 @@ class FastHrtime : public BaseObject { std::shared_ptr backing_store_; }; +static void GetFastAPIs(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + Local ret = Object::New(env->isolate()); + ret->Set(env->context(), + FIXED_ONE_BYTE_STRING(env->isolate(), "hrtime"), + FastHrtime::New(env)) + .ToChecked(); + args.GetReturnValue().Set(ret); +} + static void InitializeProcessMethods(Local target, Local unused, Local context, @@ -534,12 +544,7 @@ static void InitializeProcessMethods(Local target, env->SetMethod(target, "reallyExit", ReallyExit); env->SetMethodNoSideEffect(target, "uptime", Uptime); env->SetMethod(target, "patchProcessObject", PatchProcessObject); - - target - ->Set(env->context(), - FIXED_ONE_BYTE_STRING(env->isolate(), "hrtime"), - FastHrtime::New(env)) - .ToChecked(); + env->SetMethod(target, "getFastAPIs", GetFastAPIs); } } // namespace node diff --git a/test/cctest/test_base_object_ptr.cc b/test/cctest/test_base_object_ptr.cc index 9006ea2a3ba519..a3a8df98a9efc7 100644 --- a/test/cctest/test_base_object_ptr.cc +++ b/test/cctest/test_base_object_ptr.cc @@ -16,7 +16,7 @@ using v8::Object; // Environments may come with existing BaseObject instances. // This variable offsets the expected BaseObject counts. -static const int BASE_OBJECT_COUNT = 1; +static const int BASE_OBJECT_COUNT = 0; class BaseObjectPtrTest : public EnvironmentTestFixture {}; From 16f7545b5b62e7298f328fefd3121166227b91b7 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Tue, 14 Jul 2020 23:36:56 +0800 Subject: [PATCH 09/13] lib: initialize instance members in class constructors Since V8 snapshot does not currently support instance member initialization, initialize them in ordianry class constructors for now so that these classes can be included in the snapshot. This may be reverted once https://bugs.chromium.org/p/v8/issues/detail?id=10704 is fixed and backported. --- lib/internal/abort_controller.js | 11 ++-- lib/internal/event_target.js | 98 ++++++++++++++++---------------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/lib/internal/abort_controller.js b/lib/internal/abort_controller.js index b4ba6dd4f87603..45ffb7aee8db6a 100644 --- a/lib/internal/abort_controller.js +++ b/lib/internal/abort_controller.js @@ -55,15 +55,18 @@ function abortSignal(signal) { signal.dispatchEvent(event); } +// TODO(joyeecheung): V8 snapshot does not support instance member +// initializers for now: +// https://bugs.chromium.org/p/v8/issues/detail?id=10704 +const kSignal = Symbol('signal'); class AbortController { - #signal = new AbortSignal(); - constructor() { + this[kSignal] = new AbortSignal(); emitExperimentalWarning('AbortController'); } - get signal() { return this.#signal; } - abort() { abortSignal(this.#signal); } + get signal() { return this[kSignal]; } + abort() { abortSignal(this[kSignal]); } [customInspectSymbol](depth, options) { return customInspect(this, { diff --git a/lib/internal/event_target.js b/lib/internal/event_target.js index 983e2671b7bb6e..02ea2b8f0fff48 100644 --- a/lib/internal/event_target.js +++ b/lib/internal/event_target.js @@ -42,32 +42,30 @@ function lazyNow() { return perf_hooks.performance.now(); } +// TODO(joyeecheung): V8 snapshot does not support instance member +// initializers for now: +// https://bugs.chromium.org/p/v8/issues/detail?id=10704 +const kType = Symbol('type'); +const kDefaultPrevented = Symbol('defaultPrevented'); +const kCancelable = Symbol('cancelable'); +const kTimestamp = Symbol('timestamp'); +const kBubbles = Symbol('bubbles'); +const kComposed = Symbol('composed'); +const kPropagationStopped = Symbol('propagationStopped'); class Event { - #type = undefined; - #defaultPrevented = false; - #cancelable = false; - #timestamp = lazyNow(); - - // None of these are currently used in the Node.js implementation - // of EventTarget because there is no concept of bubbling or - // composition. We preserve their values in Event but they are - // non-ops and do not carry any semantics in Node.js - #bubbles = false; - #composed = false; - #propagationStopped = false; - - constructor(type, options) { if (arguments.length === 0) throw new ERR_MISSING_ARGS('type'); if (options != null) validateObject(options, 'options'); const { cancelable, bubbles, composed } = { ...options }; - this.#cancelable = !!cancelable; - this.#bubbles = !!bubbles; - this.#composed = !!composed; - this.#type = `${type}`; - this.#propagationStopped = false; + this[kCancelable] = !!cancelable; + this[kBubbles] = !!bubbles; + this[kComposed] = !!composed; + this[kType] = `${type}`; + this[kDefaultPrevented] = false; + this[kTimestamp] = lazyNow(); + this[kPropagationStopped] = false; // isTrusted is special (LegacyUnforgeable) Object.defineProperty(this, 'isTrusted', { get() { return false; }, @@ -87,10 +85,10 @@ class Event { }); return `${name} ${inspect({ - type: this.#type, - defaultPrevented: this.#defaultPrevented, - cancelable: this.#cancelable, - timeStamp: this.#timestamp, + type: this[kType], + defaultPrevented: this[kDefaultPrevented], + cancelable: this[kCancelable], + timeStamp: this[kTimestamp], }, opts)}`; } @@ -99,20 +97,22 @@ class Event { } preventDefault() { - this.#defaultPrevented = true; + this[kDefaultPrevented] = true; } get target() { return this[kTarget]; } get currentTarget() { return this[kTarget]; } get srcElement() { return this[kTarget]; } - get type() { return this.#type; } + get type() { return this[kType]; } - get cancelable() { return this.#cancelable; } + get cancelable() { return this[kCancelable]; } - get defaultPrevented() { return this.#cancelable && this.#defaultPrevented; } + get defaultPrevented() { + return this[kCancelable] && this[kDefaultPrevented]; + } - get timeStamp() { return this.#timestamp; } + get timeStamp() { return this[kTimestamp]; } // The following are non-op and unused properties/methods from Web API Event. @@ -121,19 +121,19 @@ class Event { composedPath() { return this[kTarget] ? [this[kTarget]] : []; } get returnValue() { return !this.defaultPrevented; } - get bubbles() { return this.#bubbles; } - get composed() { return this.#composed; } + get bubbles() { return this[kBubbles]; } + get composed() { return this[kComposed]; } get eventPhase() { return this[kTarget] ? Event.AT_TARGET : Event.NONE; } - get cancelBubble() { return this.#propagationStopped; } + get cancelBubble() { return this[kPropagationStopped]; } set cancelBubble(value) { if (value) { this.stopPropagation(); } } stopPropagation() { - this.#propagationStopped = true; + this[kPropagationStopped] = true; } static NONE = 0; @@ -157,15 +157,8 @@ Object.defineProperty(Event.prototype, SymbolToStringTag, { // the linked list makes dispatching faster, even if adding/removing is // slower. class Listener { - next; - previous; - listener; - callback; - once; - capture; - passive; - constructor(previous, listener, once, capture, passive) { + this.next = undefined; if (previous !== undefined) previous.next = this; this.previous = previous; @@ -197,7 +190,9 @@ class EventTarget { // symbol as EventTarget may be used cross-realm. See discussion in #33661. static [kIsEventTarget] = true; - [kEvents] = new Map(); + constructor() { + this[kEvents] = new Map(); + } [kNewListener](size, type, listener, once, capture, passive) {} [kRemoveListener](size, type, listener, capture) {} @@ -355,17 +350,22 @@ Object.defineProperty(EventTarget.prototype, SymbolToStringTag, { value: 'EventTarget', }); +const kMaxListeners = Symbol('maxListeners'); +const kMaxListenersWarned = Symbol('maxListenersWarned'); class NodeEventTarget extends EventTarget { static defaultMaxListeners = 10; - #maxListeners = NodeEventTarget.defaultMaxListeners; - #maxListenersWarned = false; + constructor() { + super(); + this[kMaxListeners] = NodeEventTarget.defaultMaxListeners; + this[kMaxListenersWarned] = false; + } [kNewListener](size, type, listener, once, capture, passive) { - if (this.#maxListeners > 0 && - size > this.#maxListeners && - !this.#maxListenersWarned) { - this.#maxListenersWarned = true; + if (this[kMaxListeners] > 0 && + size > this[kMaxListeners] && + !this[kMaxListenersWarned]) { + this[kMaxListenersWarned] = true; // No error code for this since it is a Warning // eslint-disable-next-line no-restricted-syntax const w = new Error('Possible EventTarget memory leak detected. ' + @@ -382,12 +382,12 @@ class NodeEventTarget extends EventTarget { setMaxListeners(n) { validateInteger(n, 'n', 0); - this.#maxListeners = n; + this[kMaxListeners] = n; return this; } getMaxListeners() { - return this.#maxListeners; + return this[kMaxListeners]; } eventNames() { From 720abb860146adcc2f061e9996bcc688a63b3840 Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Wed, 22 Apr 2020 02:57:16 +0800 Subject: [PATCH 10/13] src: snapshot node This runs `lib/internal/bootstrap/node.js` before creating the builtin snapshot and deserialize the loaders from the snapshot in deserialization mode. --- src/async_wrap.cc | 22 +++++++++++++++++ src/async_wrap.h | 2 ++ src/handle_wrap.cc | 11 +++++++++ src/handle_wrap.h | 2 ++ src/inspector_js_api.cc | 30 ++++++++++++++++++++++- src/node_buffer.cc | 39 ++++++++++++++++++++++++++++++ src/node_credentials.cc | 22 +++++++++++++++++ src/node_env_var.cc | 11 +++++++++ src/node_errors.cc | 12 ++++++++- src/node_external_reference.h | 33 +++++++++++++++++++++++-- src/node_i18n.cc | 13 ++++++++++ src/node_i18n.h | 1 - src/node_main_instance.cc | 4 --- src/node_messaging.cc | 18 +++++++++++++- src/node_process_methods.cc | 29 ++++++++++++++++++++++ src/node_task_queue.cc | 10 ++++++++ src/node_trace_events.cc | 16 +++++++++++- src/node_types.cc | 11 +++++++++ src/node_url.cc | 11 +++++++++ src/node_util.cc | 21 +++++++++++++++- src/string_decoder.cc | 9 +++++++ src/timers.cc | 12 +++++++-- tools/snapshot/snapshot_builder.cc | 2 +- 23 files changed, 326 insertions(+), 15 deletions(-) diff --git a/src/async_wrap.cc b/src/async_wrap.cc index 09775db79a8d97..3e4cbbbcbb404a 100644 --- a/src/async_wrap.cc +++ b/src/async_wrap.cc @@ -23,6 +23,7 @@ #include "async_wrap-inl.h" #include "env-inl.h" #include "node_errors.h" +#include "node_external_reference.h" #include "tracing/traced_value.h" #include "util-inl.h" @@ -695,6 +696,25 @@ void AsyncWrap::Initialize(Local target, PromiseWrap::Initialize(env); } +void AsyncWrap::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(SetupHooks); + registry->Register(SetCallbackTrampoline); + registry->Register(PushAsyncContext); + registry->Register(PopAsyncContext); + registry->Register(ExecutionAsyncResource); + registry->Register(ClearAsyncIdStack); + registry->Register(QueueDestroyAsyncId); + registry->Register(EnablePromiseHook); + registry->Register(DisablePromiseHook); + registry->Register(RegisterDestroyHook); + registry->Register(AsyncWrapObject::New); + registry->Register(AsyncWrap::GetAsyncId); + registry->Register(AsyncWrap::AsyncReset); + registry->Register(AsyncWrap::GetProviderType); + registry->Register(PromiseWrap::GetAsyncId); + registry->Register(PromiseWrap::GetTriggerAsyncId); +} AsyncWrap::AsyncWrap(Environment* env, Local object, @@ -924,3 +944,5 @@ Local AsyncWrap::GetOwner(Environment* env, Local obj) { } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(async_wrap, node::AsyncWrap::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(async_wrap, + node::AsyncWrap::RegisterExternalReferences) diff --git a/src/async_wrap.h b/src/async_wrap.h index e202314331f33e..f0e49c3df28d94 100644 --- a/src/async_wrap.h +++ b/src/async_wrap.h @@ -106,6 +106,7 @@ namespace node { class Environment; class DestroyParam; +class ExternalReferenceRegistry; class AsyncWrap : public BaseObject { public: @@ -135,6 +136,7 @@ class AsyncWrap : public BaseObject { static v8::Local GetConstructorTemplate( Environment* env); + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); static void Initialize(v8::Local target, v8::Local unused, v8::Local context, diff --git a/src/handle_wrap.cc b/src/handle_wrap.cc index 4e039d5dbf2d37..e57369844688e3 100644 --- a/src/handle_wrap.cc +++ b/src/handle_wrap.cc @@ -22,6 +22,7 @@ #include "handle_wrap.h" #include "async_wrap-inl.h" #include "env-inl.h" +#include "node_external_reference.h" #include "util-inl.h" namespace node { @@ -152,5 +153,15 @@ Local HandleWrap::GetConstructorTemplate(Environment* env) { return tmpl; } +void HandleWrap::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(HandleWrap::Close); + registry->Register(HandleWrap::HasRef); + registry->Register(HandleWrap::Ref); + registry->Register(HandleWrap::Unref); +} } // namespace node + +NODE_MODULE_EXTERNAL_REFERENCE(handle_wrap, + node::HandleWrap::RegisterExternalReferences) diff --git a/src/handle_wrap.h b/src/handle_wrap.h index 4134b28bfc1491..47eef32940bccc 100644 --- a/src/handle_wrap.h +++ b/src/handle_wrap.h @@ -32,6 +32,7 @@ namespace node { class Environment; +class ExternalReferenceRegistry; // Rules: // @@ -77,6 +78,7 @@ class HandleWrap : public AsyncWrap { static v8::Local GetConstructorTemplate( Environment* env); + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); protected: HandleWrap(Environment* env, diff --git a/src/inspector_js_api.cc b/src/inspector_js_api.cc index c318f78646634d..12569e9791879d 100644 --- a/src/inspector_js_api.cc +++ b/src/inspector_js_api.cc @@ -2,9 +2,10 @@ #include "inspector_agent.h" #include "inspector_io.h" #include "memory_tracker-inl.h" +#include "node_external_reference.h" #include "util-inl.h" -#include "v8.h" #include "v8-inspector.h" +#include "v8.h" #include @@ -345,8 +346,35 @@ void Initialize(Local target, Local unused, } } // namespace + +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(InspectorConsoleCall); + registry->Register(SetConsoleExtensionInstaller); + registry->Register(CallAndPauseOnStart); + registry->Register(Open); + registry->Register(Url); + registry->Register(WaitForDebugger); + + registry->Register(AsyncTaskScheduledWrapper); + registry->Register(InvokeAsyncTaskFnWithId<&Agent::AsyncTaskCanceled>); + registry->Register(InvokeAsyncTaskFnWithId<&Agent::AsyncTaskStarted>); + registry->Register(InvokeAsyncTaskFnWithId<&Agent::AsyncTaskFinished>); + + registry->Register(RegisterAsyncHookWrapper); + registry->Register(IsEnabled); + + registry->Register(JSBindingsConnection::New); + registry->Register(JSBindingsConnection::Dispatch); + registry->Register(JSBindingsConnection::Disconnect); + registry->Register(JSBindingsConnection::New); + registry->Register(JSBindingsConnection::Dispatch); + registry->Register(JSBindingsConnection::Disconnect); +} + } // namespace inspector } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(inspector, node::inspector::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(inspector, + node::inspector::RegisterExternalReferences) diff --git a/src/node_buffer.cc b/src/node_buffer.cc index 2028849775a98c..b2fa0ef20174a9 100644 --- a/src/node_buffer.cc +++ b/src/node_buffer.cc @@ -23,6 +23,7 @@ #include "allocated_buffer-inl.h" #include "node.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" #include "env-inl.h" @@ -1197,7 +1198,45 @@ void Initialize(Local target, } } // anonymous namespace + +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(SetBufferPrototype); + registry->Register(CreateFromString); + + registry->Register(ByteLengthUtf8); + registry->Register(Copy); + registry->Register(Compare); + registry->Register(CompareOffset); + registry->Register(Fill); + registry->Register(IndexOfBuffer); + registry->Register(IndexOfNumber); + registry->Register(IndexOfString); + + registry->Register(Swap16); + registry->Register(Swap32); + registry->Register(Swap64); + + registry->Register(EncodeInto); + registry->Register(EncodeUtf8String); + + registry->Register(StringSlice); + registry->Register(StringSlice); + registry->Register(StringSlice); + registry->Register(StringSlice); + registry->Register(StringSlice); + registry->Register(StringSlice); + + registry->Register(StringWrite); + registry->Register(StringWrite); + registry->Register(StringWrite); + registry->Register(StringWrite); + registry->Register(StringWrite); + registry->Register(StringWrite); + registry->Register(GetZeroFillToggle); +} + } // namespace Buffer } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(buffer, node::Buffer::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(buffer, node::Buffer::RegisterExternalReferences) diff --git a/src/node_credentials.cc b/src/node_credentials.cc index 83db705e10df4f..acc48cac3c90ef 100644 --- a/src/node_credentials.cc +++ b/src/node_credentials.cc @@ -1,4 +1,5 @@ #include "env-inl.h" +#include "node_external_reference.h" #include "node_internals.h" #include "util-inl.h" @@ -371,6 +372,25 @@ static void InitGroups(const FunctionCallbackInfo& args) { #endif // NODE_IMPLEMENTS_POSIX_CREDENTIALS +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(SafeGetenv); + +#ifdef NODE_IMPLEMENTS_POSIX_CREDENTIALS + registry->Register(GetUid); + registry->Register(GetEUid); + registry->Register(GetGid); + registry->Register(GetEGid); + registry->Register(GetGroups); + + registry->Register(InitGroups); + registry->Register(SetEGid); + registry->Register(SetEUid); + registry->Register(SetGid); + registry->Register(SetUid); + registry->Register(SetGroups); +#endif // NODE_IMPLEMENTS_POSIX_CREDENTIALS +} + static void Initialize(Local target, Local unused, Local context, @@ -403,3 +423,5 @@ static void Initialize(Local target, } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(credentials, node::credentials::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(credentials, + node::credentials::RegisterExternalReferences) diff --git a/src/node_env_var.cc b/src/node_env_var.cc index 1a162888fd37a7..1cf959a6095afd 100644 --- a/src/node_env_var.cc +++ b/src/node_env_var.cc @@ -1,6 +1,7 @@ #include "debug_utils-inl.h" #include "env-inl.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_process.h" #include // tzset(), _tzset() @@ -384,4 +385,14 @@ MaybeLocal CreateEnvVarProxy(Local context, Isolate* isolate) { PropertyHandlerFlags::kHasNoSideEffect)); return scope.EscapeMaybe(env_proxy_template->NewInstance(context)); } + +void RegisterEnvVarExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(EnvGetter); + registry->Register(EnvSetter); + registry->Register(EnvQuery); + registry->Register(EnvDeleter); + registry->Register(EnvEnumerator); +} } // namespace node + +NODE_MODULE_EXTERNAL_REFERENCE(env_var, node::RegisterEnvVarExternalReferences) diff --git a/src/node_errors.cc b/src/node_errors.cc index 813bc3fd8e0eb7..dc6d20b500c655 100644 --- a/src/node_errors.cc +++ b/src/node_errors.cc @@ -3,9 +3,10 @@ #include "debug_utils-inl.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" -#include "node_report.h" #include "node_process.h" +#include "node_report.h" #include "node_v8_platform-inl.h" #include "util-inl.h" @@ -852,6 +853,14 @@ static void TriggerUncaughtException(const FunctionCallbackInfo& args) { errors::TriggerUncaughtException(isolate, exception, message, from_promise); } +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(SetPrepareStackTraceCallback); + registry->Register(EnableSourceMaps); + registry->Register(SetEnhanceStackForFatalException); + registry->Register(NoSideEffectsToString); + registry->Register(TriggerUncaughtException); +} + void Initialize(Local target, Local unused, Local context, @@ -1023,3 +1032,4 @@ void TriggerUncaughtException(Isolate* isolate, const v8::TryCatch& try_catch) { } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(errors, node::errors::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(errors, node::errors::RegisterExternalReferences) diff --git a/src/node_external_reference.h b/src/node_external_reference.h index 8c97eca2264a60..f332103c3f5a27 100644 --- a/src/node_external_reference.h +++ b/src/node_external_reference.h @@ -47,12 +47,41 @@ class ExternalReferenceRegistry { }; #define EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) \ + V(async_wrap) \ V(binding) \ + V(buffer) \ + V(credentials) \ + V(env_var) \ + V(errors) \ + V(handle_wrap) \ + V(messaging) \ V(native_module) \ - V(process_object) + V(process_methods) \ + V(process_object) \ + V(task_queue) \ + V(url) \ + V(util) \ + V(string_decoder) \ + V(trace_events) \ + V(timers) \ + V(types) + +#if NODE_HAVE_I18N_SUPPORT +#define EXTERNAL_REFERENCE_BINDING_LIST_I18N(V) V(icu) +#else +#define EXTERNAL_REFERENCE_BINDING_LIST_I18N(V) +#endif // NODE_HAVE_I18N_SUPPORT + +#if HAVE_INSPECTOR +#define EXTERNAL_REFERENCE_BINDING_LIST_INSPECTOR(V) V(inspector) +#else +#define EXTERNAL_REFERENCE_BINDING_LIST_INSPECTOR(V) +#endif // HAVE_INSPECTOR #define EXTERNAL_REFERENCE_BINDING_LIST(V) \ - EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) + EXTERNAL_REFERENCE_BINDING_LIST_BASE(V) \ + EXTERNAL_REFERENCE_BINDING_LIST_INSPECTOR(V) \ + EXTERNAL_REFERENCE_BINDING_LIST_I18N(V) } // namespace node diff --git a/src/node_i18n.cc b/src/node_i18n.cc index cc2d245b8a77d9..48f95ceb68d9c1 100644 --- a/src/node_i18n.cc +++ b/src/node_i18n.cc @@ -41,6 +41,7 @@ #include "node_i18n.h" +#include "node_external_reference.h" #if defined(NODE_HAVE_I18N_SUPPORT) @@ -824,9 +825,21 @@ void Initialize(Local target, env->SetMethod(target, "hasConverter", ConverterObject::Has); } +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(ToUnicode); + registry->Register(ToASCII); + registry->Register(GetStringWidth); + registry->Register(ICUErrorName); + registry->Register(Transcode); + registry->Register(ConverterObject::Create); + registry->Register(ConverterObject::Decode); + registry->Register(ConverterObject::Has); +} + } // namespace i18n } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(icu, node::i18n::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(icu, node::i18n::RegisterExternalReferences) #endif // NODE_HAVE_I18N_SUPPORT diff --git a/src/node_i18n.h b/src/node_i18n.h index 5c1501ea1908e0..22164e6647d913 100644 --- a/src/node_i18n.h +++ b/src/node_i18n.h @@ -36,7 +36,6 @@ #include namespace node { - namespace i18n { bool InitializeICUDirectory(const std::string& path); diff --git a/src/node_main_instance.cc b/src/node_main_instance.cc index 617a71e7cac60d..14e0d49ce959f0 100644 --- a/src/node_main_instance.cc +++ b/src/node_main_instance.cc @@ -263,10 +263,6 @@ NodeMainInstance::CreateMainEnvironment(int* exit_code, return nullptr; } - if (deserialize_mode_ && env->BootstrapNode().IsEmpty()) { - return nullptr; - } - CHECK(env->req_wrap_queue()->IsEmpty()); CHECK(env->handle_wrap_queue()->IsEmpty()); env->set_has_run_bootstrapping_code(true); diff --git a/src/node_messaging.cc b/src/node_messaging.cc index 3da2358ce8dc50..add851e0836f3a 100644 --- a/src/node_messaging.cc +++ b/src/node_messaging.cc @@ -3,9 +3,10 @@ #include "async_wrap-inl.h" #include "debug_utils-inl.h" #include "memory_tracker-inl.h" -#include "node_contextify.h" #include "node_buffer.h" +#include "node_contextify.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_process.h" #include "util-inl.h" @@ -1352,9 +1353,24 @@ static void InitMessaging(Local target, } } +static void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(MessageChannel); + registry->Register(JSTransferable::New); + registry->Register(MessagePort::New); + registry->Register(MessagePort::PostMessage); + registry->Register(MessagePort::Start); + registry->Register(MessagePort::Stop); + registry->Register(MessagePort::Drain); + registry->Register(MessagePort::ReceiveMessage); + registry->Register(MessagePort::MoveToContext); + registry->Register(SetDeserializerCreateObjectFunction); +} + } // anonymous namespace } // namespace worker } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(messaging, node::worker::InitMessaging) +NODE_MODULE_EXTERNAL_REFERENCE(messaging, + node::worker::RegisterExternalReferences) diff --git a/src/node_process_methods.cc b/src/node_process_methods.cc index e714cdd30f99b1..105cbff151b151 100644 --- a/src/node_process_methods.cc +++ b/src/node_process_methods.cc @@ -4,6 +4,7 @@ #include "memory_tracker-inl.h" #include "node.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" #include "node_process.h" #include "util-inl.h" @@ -547,6 +548,32 @@ static void InitializeProcessMethods(Local target, env->SetMethod(target, "getFastAPIs", GetFastAPIs); } +void RegisterProcessMethodsExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(DebugProcess); + registry->Register(DebugEnd); + registry->Register(Abort); + registry->Register(CauseSegfault); + registry->Register(Chdir); + + registry->Register(Umask); + registry->Register(RawDebug); + registry->Register(MemoryUsage); + registry->Register(CPUUsage); + registry->Register(ResourceUsage); + + registry->Register(GetActiveRequests); + registry->Register(GetActiveHandles); + registry->Register(Kill); + + registry->Register(Cwd); + registry->Register(binding::DLOpen); + registry->Register(ReallyExit); + registry->Register(Uptime); + registry->Register(PatchProcessObject); + registry->Register(GetFastAPIs); +} + } // namespace node namespace v8 { @@ -562,3 +589,5 @@ class WrapperTraits { NODE_MODULE_CONTEXT_AWARE_INTERNAL(process_methods, node::InitializeProcessMethods) +NODE_MODULE_EXTERNAL_REFERENCE(process_methods, + node::RegisterProcessMethodsExternalReferences) diff --git a/src/node_task_queue.cc b/src/node_task_queue.cc index 3c7d9bae0884c5..9a22cabdedf4a2 100644 --- a/src/node_task_queue.cc +++ b/src/node_task_queue.cc @@ -1,6 +1,7 @@ #include "env-inl.h" #include "node.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_internals.h" #include "node_process.h" #include "util-inl.h" @@ -145,7 +146,16 @@ static void Initialize(Local target, SetPromiseRejectCallback); } +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(EnqueueMicrotask); + registry->Register(SetTickCallback); + registry->Register(RunMicrotasks); + registry->Register(SetPromiseRejectCallback); +} + } // namespace task_queue } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(task_queue, node::task_queue::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(task_queue, + node::task_queue::RegisterExternalReferences) diff --git a/src/node_trace_events.cc b/src/node_trace_events.cc index 58813a9083a560..9cefaa9227031d 100644 --- a/src/node_trace_events.cc +++ b/src/node_trace_events.cc @@ -2,6 +2,7 @@ #include "env-inl.h" #include "memory_tracker-inl.h" #include "node.h" +#include "node_external_reference.h" #include "node_internals.h" #include "node_v8_platform-inl.h" #include "tracing/agent.h" @@ -12,6 +13,8 @@ namespace node { +class ExternalReferenceRegistry; + using v8::Array; using v8::Context; using v8::Function; @@ -29,7 +32,7 @@ class NodeCategorySet : public BaseObject { Local unused, Local context, void* priv); - + static void RegisterExternalReferences(ExternalReferenceRegistry* registry); static void New(const FunctionCallbackInfo& args); static void Enable(const FunctionCallbackInfo& args); static void Disable(const FunctionCallbackInfo& args); @@ -154,7 +157,18 @@ void NodeCategorySet::Initialize(Local target, binding->Get(context, trace).ToLocalChecked()).Check(); } +void NodeCategorySet::RegisterExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(GetEnabledCategories); + registry->Register(SetTraceCategoryStateUpdateHandler); + registry->Register(NodeCategorySet::New); + registry->Register(NodeCategorySet::Enable); + registry->Register(NodeCategorySet::Disable); +} + } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(trace_events, node::NodeCategorySet::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE( + trace_events, node::NodeCategorySet::RegisterExternalReferences) diff --git a/src/node_types.cc b/src/node_types.cc index 9643a66668144e..1889d8c304110b 100644 --- a/src/node_types.cc +++ b/src/node_types.cc @@ -1,5 +1,6 @@ #include "env-inl.h" #include "node.h" +#include "node_external_reference.h" using v8::Context; using v8::FunctionCallbackInfo; @@ -77,6 +78,16 @@ void InitializeTypes(Local target, } } // anonymous namespace + +void RegisterTypesExternalReferences(ExternalReferenceRegistry* registry) { +#define V(type) registry->Register(Is##type); + VALUE_METHOD_MAP(V) +#undef V + + registry->Register(IsAnyArrayBuffer); + registry->Register(IsBoxedPrimitive); +} } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(types, node::InitializeTypes) +NODE_MODULE_EXTERNAL_REFERENCE(types, node::RegisterTypesExternalReferences) diff --git a/src/node_url.cc b/src/node_url.cc index a4549ea4b8c38e..efab1060f883cc 100644 --- a/src/node_url.cc +++ b/src/node_url.cc @@ -1,6 +1,7 @@ #include "node_url.h" #include "base_object-inl.h" #include "node_errors.h" +#include "node_external_reference.h" #include "node_i18n.h" #include "util-inl.h" @@ -2325,6 +2326,15 @@ void Initialize(Local target, } } // namespace +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(Parse); + registry->Register(EncodeAuthSet); + registry->Register(ToUSVString); + registry->Register(DomainToASCII); + registry->Register(DomainToUnicode); + registry->Register(SetURLConstructor); +} + std::string URL::ToFilePath() const { if (context_.scheme != "file:") { return ""; @@ -2446,3 +2456,4 @@ MaybeLocal URL::ToObject(Environment* env) const { } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(url, node::url::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(url, node::url::RegisterExternalReferences) diff --git a/src/node_util.cc b/src/node_util.cc index 22a372ad09a3bd..44148ba2b0958a 100644 --- a/src/node_util.cc +++ b/src/node_util.cc @@ -1,6 +1,7 @@ +#include "base_object-inl.h" #include "node_errors.h" +#include "node_external_reference.h" #include "util-inl.h" -#include "base_object-inl.h" namespace node { namespace util { @@ -262,6 +263,23 @@ static void GuessHandleType(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(OneByteString(env->isolate(), type)); } +void RegisterExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(GetHiddenValue); + registry->Register(SetHiddenValue); + registry->Register(GetPromiseDetails); + registry->Register(GetProxyDetails); + registry->Register(PreviewEntries); + registry->Register(GetOwnNonIndexProperties); + registry->Register(GetConstructorName); + registry->Register(Sleep); + registry->Register(ArrayBufferViewHasBuffer); + registry->Register(WeakReference::New); + registry->Register(WeakReference::Get); + registry->Register(WeakReference::IncRef); + registry->Register(WeakReference::DecRef); + registry->Register(GuessHandleType); +} + void Initialize(Local target, Local unused, Local context, @@ -339,3 +357,4 @@ void Initialize(Local target, } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(util, node::util::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(util, node::util::RegisterExternalReferences) diff --git a/src/string_decoder.cc b/src/string_decoder.cc index 6ec84e0e11ed31..a25c903987016f 100644 --- a/src/string_decoder.cc +++ b/src/string_decoder.cc @@ -3,6 +3,7 @@ #include "env-inl.h" #include "node_buffer.h" +#include "node_external_reference.h" #include "string_bytes.h" #include "util.h" @@ -322,7 +323,15 @@ void InitializeStringDecoder(Local target, } // anonymous namespace +void RegisterStringDecoderExternalReferences( + ExternalReferenceRegistry* registry) { + registry->Register(DecodeData); + registry->Register(FlushData); +} + } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(string_decoder, node::InitializeStringDecoder) +NODE_MODULE_EXTERNAL_REFERENCE(string_decoder, + node::RegisterStringDecoderExternalReferences) diff --git a/src/timers.cc b/src/timers.cc index fab1b12018a921..5014f2c66a5d24 100644 --- a/src/timers.cc +++ b/src/timers.cc @@ -1,4 +1,5 @@ #include "env-inl.h" +#include "node_external_reference.h" #include "util-inl.h" #include "v8.h" @@ -57,9 +58,16 @@ void Initialize(Local target, FIXED_ONE_BYTE_STRING(env->isolate(), "immediateInfo"), env->immediate_info()->fields().GetJSArray()).Check(); } - - } // anonymous namespace +void RegisterTimerExternalReferences(ExternalReferenceRegistry* registry) { + registry->Register(GetLibuvNow); + registry->Register(SetupTimers); + registry->Register(ScheduleTimer); + registry->Register(ToggleTimerRef); + registry->Register(ToggleImmediateRef); +} + } // namespace node NODE_MODULE_CONTEXT_AWARE_INTERNAL(timers, node::Initialize) +NODE_MODULE_EXTERNAL_REFERENCE(timers, node::RegisterTimerExternalReferences) diff --git a/tools/snapshot/snapshot_builder.cc b/tools/snapshot/snapshot_builder.cc index 364202711105a4..94fe1604149c43 100644 --- a/tools/snapshot/snapshot_builder.cc +++ b/tools/snapshot/snapshot_builder.cc @@ -130,7 +130,7 @@ std::string SnapshotBuilder::Generate( nullptr, node::EnvironmentFlags::kDefaultFlags, {}); - env->BootstrapInternalLoaders().ToLocalChecked(); + env->RunBootstrapping().ToLocalChecked(); if (per_process::enabled_debug_list.enabled(DebugCategory::MKSNAPSHOT)) { env->PrintAllBaseObjects(); printf("Environment = %p\n", env); From 393f6534f3731e139778c03ecb52bfe40cd7ec1a Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 17 Jul 2020 03:15:38 +0800 Subject: [PATCH 11/13] fixup! src: snapshot Environment upon instantiation --- src/node.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/node.cc b/src/node.cc index c7da03616e65f9..f14ff44be80d74 100644 --- a/src/node.cc +++ b/src/node.cc @@ -1061,7 +1061,6 @@ int Start(int argc, char** argv) { { Isolate::CreateParams params; const std::vector* indexes = nullptr; - std::vector external_references; const EnvSerializeInfo* env_info = nullptr; bool force_no_snapshot = per_process::cli_options->per_isolate->no_node_snapshot; From f85cf424ba0f91618dfc2a46bf632e401bdc761e Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Fri, 17 Jul 2020 03:15:53 +0800 Subject: [PATCH 12/13] fixup! src: snapshot loaders --- src/node_native_module_env.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/node_native_module_env.cc b/src/node_native_module_env.cc index f388c696b0ad91..b59ed7cb75cf67 100644 --- a/src/node_native_module_env.cc +++ b/src/node_native_module_env.cc @@ -221,7 +221,6 @@ void NativeModuleEnv::RegisterExternalReferences( ExternalReferenceRegistry* registry) { registry->Register(ConfigStringGetter); registry->Register(ModuleIdsGetter); - registry->Register(ModuleIdsGetter); registry->Register(GetModuleCategories); registry->Register(GetCacheUsage); registry->Register(CompileFunction); From 8c9260c3e37c4f95962e51e55eefbb62faf9dcbb Mon Sep 17 00:00:00 2001 From: Joyee Cheung Date: Sat, 18 Jul 2020 07:49:40 +0800 Subject: [PATCH 13/13] fixup! src: snapshot Environment upon instantiation --- src/env.cc | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/env.cc b/src/env.cc index 0f9c1a326740b6..1cf37d4fa91c8c 100644 --- a/src/env.cc +++ b/src/env.cc @@ -1072,7 +1072,7 @@ std::ostream& operator<<(std::ostream& output, for (const SnapshotIndex i : v) { output << i << ", "; } - output << " }\n"; + output << " }"; return output; } @@ -1323,14 +1323,15 @@ void Environment::DeserializeProperties(const EnvSerializeInfo* info) { if (templates.size() > i && id == templates[i].id) { \ const PropInfo& d = templates[i]; \ DCHECK_EQ(d.name, #PropertyName); \ - MaybeLocal field = \ + MaybeLocal maybe_field = \ isolate_->GetDataFromSnapshotOnce(d.index); \ - if (field.IsEmpty()) { \ + Local field; \ + if (!maybe_field.ToLocal(&field)) { \ fprintf(stderr, \ "Failed to deserialize environment template " #PropertyName \ "\n"); \ } \ - set_##PropertyName(field.ToLocalChecked()); \ + set_##PropertyName(field); \ i++; \ } \ } while (0); \ @@ -1346,14 +1347,15 @@ void Environment::DeserializeProperties(const EnvSerializeInfo* info) { if (values.size() > i && id == values[i].id) { \ const PropInfo& d = values[i]; \ DCHECK_EQ(d.name, #PropertyName); \ - MaybeLocal field = \ + MaybeLocal maybe_field = \ ctx->GetDataFromSnapshotOnce(d.index); \ - if (field.IsEmpty()) { \ + Local field; \ + if (!maybe_field.ToLocal(&field)) { \ fprintf(stderr, \ "Failed to deserialize environment value " #PropertyName \ "\n"); \ } \ - set_##PropertyName(field.ToLocalChecked()); \ + set_##PropertyName(field); \ i++; \ } \ } while (0); \ @@ -1361,13 +1363,14 @@ void Environment::DeserializeProperties(const EnvSerializeInfo* info) { ENVIRONMENT_STRONG_PERSISTENT_VALUES(V); #undef V - MaybeLocal ctx_from_snapshot = + MaybeLocal maybe_ctx_from_snapshot = ctx->GetDataFromSnapshotOnce(info->context); - if (ctx_from_snapshot.IsEmpty()) { + Local ctx_from_snapshot; + if (!maybe_ctx_from_snapshot.ToLocal(&ctx_from_snapshot)) { fprintf(stderr, "Failed to deserialize context back reference from the snapshot\n"); } - CHECK_EQ(ctx_from_snapshot.ToLocalChecked(), ctx); + CHECK_EQ(ctx_from_snapshot, ctx); } void Environment::BuildEmbedderGraph(Isolate* isolate,