From 2a56d7e62976cfdde787917da7ebfe10864af9c8 Mon Sep 17 00:00:00 2001 From: theanarkh Date: Sat, 27 Aug 2022 23:01:46 +0800 Subject: [PATCH] v8: add setHeapSnapshotNearHeapLimit --- doc/api/v8.md | 13 ++ lib/v8.js | 20 ++- src/env.cc | 6 +- src/env.h | 9 ++ src/node.cc | 2 +- src/node_v8.cc | 17 +++ .../workload/grow-and-set-near-heap-limit.js | 7 + .../grow-worker-and-set-near-heap-limit.js | 15 +++ ...apshot-near-heap-limit-by-api-in-worker.js | 41 ++++++ ...est-heapsnapshot-near-heap-limit-by-api.js | 121 ++++++++++++++++++ .../test-heapsnapshot-near-heap-limit.js | 2 +- 11 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/workload/grow-and-set-near-heap-limit.js create mode 100644 test/fixtures/workload/grow-worker-and-set-near-heap-limit.js create mode 100644 test/parallel/test-heapsnapshot-near-heap-limit-by-api-in-worker.js create mode 100644 test/pummel/test-heapsnapshot-near-heap-limit-by-api.js diff --git a/doc/api/v8.md b/doc/api/v8.md index 02e52619670482..caa5e70b1abced 100644 --- a/doc/api/v8.md +++ b/doc/api/v8.md @@ -356,6 +356,18 @@ if (isMainThread) { } ``` +## `v8.setHeapSnapshotNearHeapLimit(limit)` + + + +> Stability: 1 - Experimental + +The API is a no-op if `--heapsnapshot-near-heap-limit` is already set from the +command line or the API is called twice. And `limit` must be greater than or +equal to 1. See [`--heapsnapshot-near-heap-limit`][] for more information. + ## Serialization API The serialization API provides means of serializing JavaScript values in a way @@ -1020,6 +1032,7 @@ Returns true if the Node.js instance is run to build a snapshot. [HTML structured clone algorithm]: https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm [Hook Callbacks]: #hook-callbacks [V8]: https://developers.google.com/v8/ +[`--heapsnapshot-near-heap-limit`]: cli.md#--heapsnapshot-near-heap-limitmax_count [`AsyncLocalStorage`]: async_context.md#class-asynclocalstorage [`Buffer`]: buffer.md [`DefaultDeserializer`]: #class-v8defaultdeserializer diff --git a/lib/v8.js b/lib/v8.js index db3dba59565458..ad1e4bf04a38df 100644 --- a/lib/v8.js +++ b/lib/v8.js @@ -33,7 +33,7 @@ const { } = primordials; const { Buffer } = require('buffer'); -const { validateString } = require('internal/validators'); +const { validateString, validateNumber } = require('internal/validators'); const { Serializer, Deserializer @@ -59,7 +59,7 @@ const { } = internalBinding('heap_utils'); const { HeapSnapshotStream } = require('internal/heap_utils'); const promiseHooks = require('internal/promise_hooks'); - +const { getOptionValue } = require('internal/options'); /** * Generates a snapshot of the current V8 heap * and writes it to a JSON file. @@ -95,6 +95,7 @@ const { updateHeapStatisticsBuffer, updateHeapSpaceStatisticsBuffer, updateHeapCodeStatisticsBuffer, + setHeapSnapshotNearHeapLimit: _setHeapSnapshotNearHeapLimit, // Properties for heap statistics buffer extraction. kTotalHeapSizeIndex, @@ -226,6 +227,18 @@ function getHeapCodeStatistics() { }; } +let called = false; +function setHeapSnapshotNearHeapLimit(limit) { + validateNumber(limit, 'limit', 1); + const value = getOptionValue('--heapsnapshot-near-heap-limit'); + // Can not be called twice. + if (value > 0 || called) { + return; + } + called = true; + _setHeapSnapshotNearHeapLimit(limit); +} + /* V8 serialization API */ /* JS methods for the base objects */ @@ -387,5 +400,6 @@ module.exports = { serialize, writeHeapSnapshot, promiseHooks, - startupSnapshot + startupSnapshot, + setHeapSnapshotNearHeapLimit, }; diff --git a/src/env.cc b/src/env.cc index 9bc9346e653fe1..212e49e9f5c99c 100644 --- a/src/env.cc +++ b/src/env.cc @@ -720,6 +720,8 @@ Environment::Environment(IsolateData* isolate_data, inspector_host_port_ = std::make_shared>( options_->debug_options().host_port); + set_heap_snapshot_near_heap_limit(options_->heap_snapshot_near_heap_limit); + if (!(flags_ & EnvironmentFlags::kOwnsProcessState)) { set_abort_on_uncaught_exception(false); } @@ -834,7 +836,7 @@ Environment::~Environment() { // FreeEnvironment() should have set this. CHECK(is_stopping()); - if (options_->heap_snapshot_near_heap_limit > heap_limit_snapshot_taken_) { + if (get_heap_snapshot_near_heap_limit() > heap_limit_snapshot_taken_) { isolate_->RemoveNearHeapLimitCallback(Environment::NearHeapLimitCallback, 0); } @@ -1982,7 +1984,7 @@ size_t Environment::NearHeapLimitCallback(void* data, // Don't take more snapshots than the number specified by // --heapsnapshot-near-heap-limit. if (env->heap_limit_snapshot_taken_ < - env->options_->heap_snapshot_near_heap_limit) { + env->get_heap_snapshot_near_heap_limit()) { env->isolate()->AddNearHeapLimitCallback(NearHeapLimitCallback, env); } diff --git a/src/env.h b/src/env.h index 6f148dbd041b83..f25f62c3bf8337 100644 --- a/src/env.h +++ b/src/env.h @@ -1489,6 +1489,14 @@ class Environment : public MemoryRetainer { template void ForEachBaseObject(T&& iterator); + inline int64_t get_heap_snapshot_near_heap_limit() { + return heap_snapshot_near_heap_limit_; + } + + inline void set_heap_snapshot_near_heap_limit(int64_t limit) { + heap_snapshot_near_heap_limit_ = limit; + } + private: inline void ThrowError(v8::Local (*fun)(v8::Local), const char* errmsg); @@ -1547,6 +1555,7 @@ class Environment : public MemoryRetainer { bool is_processing_heap_limit_callback_ = false; int64_t heap_limit_snapshot_taken_ = 0; + int64_t heap_snapshot_near_heap_limit_ = 0; uint32_t module_id_counter_ = 0; uint32_t script_id_counter_ = 0; diff --git a/src/node.cc b/src/node.cc index 2891c18bb9aa9a..b07245aa7ce8fe 100644 --- a/src/node.cc +++ b/src/node.cc @@ -273,7 +273,7 @@ static void AtomicsWaitCallback(Isolate::AtomicsWaitEvent event, void Environment::InitializeDiagnostics() { isolate_->GetHeapProfiler()->AddBuildEmbedderGraphCallback( Environment::BuildEmbedderGraph, this); - if (options_->heap_snapshot_near_heap_limit > 0) { + if (get_heap_snapshot_near_heap_limit() > 0) { isolate_->AddNearHeapLimitCallback(Environment::NearHeapLimitCallback, this); } diff --git a/src/node_v8.cc b/src/node_v8.cc index 001d3464ec0e43..97e601d8868250 100644 --- a/src/node_v8.cc +++ b/src/node_v8.cc @@ -157,6 +157,18 @@ void CachedDataVersionTag(const FunctionCallbackInfo& args) { args.GetReturnValue().Set(result); } +void SetHeapSnapshotNearHeapLimit(const FunctionCallbackInfo& args) { + CHECK(args[0]->IsNumber()); + Environment* env = Environment::GetCurrent(args); + if (env->get_heap_snapshot_near_heap_limit() == 0) { + Isolate* const isolate = args.GetIsolate(); + int64_t limit = args[0].As()->Value(); + CHECK_GT(limit, 0); + env->set_heap_snapshot_near_heap_limit(limit); + isolate->AddNearHeapLimitCallback(Environment::NearHeapLimitCallback, env); + } +} + void UpdateHeapStatisticsBuffer(const FunctionCallbackInfo& args) { BindingData* data = Environment::GetBindingData(args); HeapStatistics s; @@ -212,6 +224,10 @@ void Initialize(Local target, SetMethodNoSideEffect( context, target, "cachedDataVersionTag", CachedDataVersionTag); + SetMethodNoSideEffect(context, + target, + "setHeapSnapshotNearHeapLimit", + SetHeapSnapshotNearHeapLimit); SetMethod(context, target, "updateHeapStatisticsBuffer", @@ -267,6 +283,7 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { registry->Register(UpdateHeapCodeStatisticsBuffer); registry->Register(UpdateHeapSpaceStatisticsBuffer); registry->Register(SetFlagsFromString); + registry->Register(SetHeapSnapshotNearHeapLimit); } } // namespace v8_utils diff --git a/test/fixtures/workload/grow-and-set-near-heap-limit.js b/test/fixtures/workload/grow-and-set-near-heap-limit.js new file mode 100644 index 00000000000000..a046afe057678b --- /dev/null +++ b/test/fixtures/workload/grow-and-set-near-heap-limit.js @@ -0,0 +1,7 @@ +'use strict'; +const path = require('path'); +const v8 = require('v8'); + +v8.setHeapSnapshotNearHeapLimit(+process.env.limit); + +require(path.resolve(__dirname, 'grow.js')); diff --git a/test/fixtures/workload/grow-worker-and-set-near-heap-limit.js b/test/fixtures/workload/grow-worker-and-set-near-heap-limit.js new file mode 100644 index 00000000000000..598088bcf90a78 --- /dev/null +++ b/test/fixtures/workload/grow-worker-and-set-near-heap-limit.js @@ -0,0 +1,15 @@ +'use strict'; +const path = require('path'); +const { Worker } = require('worker_threads'); +const max_snapshots = parseInt(process.env.TEST_SNAPSHOTS) || 1; +new Worker(path.join(__dirname, 'grow-and-set-near-heap-limit.js'), { + env: { + ...process.env, + limit: max_snapshots, + }, + resourceLimits: { + maxOldGenerationSizeMb: + parseInt(process.env.TEST_OLD_SPACE_SIZE) || 20 + } +}); + diff --git a/test/parallel/test-heapsnapshot-near-heap-limit-by-api-in-worker.js b/test/parallel/test-heapsnapshot-near-heap-limit-by-api-in-worker.js new file mode 100644 index 00000000000000..ac55933a47122b --- /dev/null +++ b/test/parallel/test-heapsnapshot-near-heap-limit-by-api-in-worker.js @@ -0,0 +1,41 @@ +// Copy from test-heapsnapshot-near-heap-limit-worker.js +'use strict'; + +require('../common'); +const tmpdir = require('../common/tmpdir'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fixtures = require('../common/fixtures'); +const fs = require('fs'); + +const env = { + ...process.env, + NODE_DEBUG_NATIVE: 'diagnostics' +}; + +{ + tmpdir.refresh(); + const child = spawnSync(process.execPath, [ + fixtures.path('workload', 'grow-worker-and-set-near-heap-limit.js'), + ], { + cwd: tmpdir.path, + env: { + TEST_SNAPSHOTS: 1, + TEST_OLD_SPACE_SIZE: 50, + ...env + } + }); + console.log(child.stdout.toString()); + const stderr = child.stderr.toString(); + console.log(stderr); + const risky = /Not generating snapshots because it's too risky/.test(stderr); + if (!risky) { + // There should be one snapshot taken and then after the + // snapshot heap limit callback is popped, the OOM callback + // becomes effective. + assert(stderr.includes('ERR_WORKER_OUT_OF_MEMORY')); + const list = fs.readdirSync(tmpdir.path) + .filter((file) => file.endsWith('.heapsnapshot')); + assert.strictEqual(list.length, 1); + } +} diff --git a/test/pummel/test-heapsnapshot-near-heap-limit-by-api.js b/test/pummel/test-heapsnapshot-near-heap-limit-by-api.js new file mode 100644 index 00000000000000..1ba6b53b9b9fcd --- /dev/null +++ b/test/pummel/test-heapsnapshot-near-heap-limit-by-api.js @@ -0,0 +1,121 @@ +// Copy from test-heapsnapshot-near-heap-limit.js +'use strict'; + +const common = require('../common'); + +if (common.isPi) { + common.skip('Too slow for Raspberry Pi devices'); +} + +const tmpdir = require('../common/tmpdir'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const fixtures = require('../common/fixtures'); +const fs = require('fs'); +const v8 = require('v8'); + +const invalidValues = [-1, '', {}, NaN, undefined]; +let errorCount = 0; +for (let i = 0; i < invalidValues.length; i++) { + try { + v8.setHeapSnapshotNearHeapLimit(invalidValues[i]); + } catch (e) { + console.log(e); + errorCount++; + } +} +assert.strictEqual(errorCount, invalidValues.length); + +// Set twice +v8.setHeapSnapshotNearHeapLimit(1); +v8.setHeapSnapshotNearHeapLimit(2); + +const env = { + ...process.env, + NODE_DEBUG_NATIVE: 'diagnostics', +}; + +{ + console.log('\nTesting set by cmd option and api'); + tmpdir.refresh(); + const child = spawnSync(process.execPath, [ + '--trace-gc', + '--heapsnapshot-near-heap-limit=1', + '--max-old-space-size=50', + fixtures.path('workload', 'grow-and-set-near-heap-limit.js'), + ], { + cwd: tmpdir.path, + env: { + ...env, + limit: 1, + }, + }); + console.log(child.stdout.toString()); + const stderr = child.stderr.toString(); + console.log(stderr); + assert(common.nodeProcessAborted(child.status, child.signal), + 'process should have aborted, but did not'); + const list = fs.readdirSync(tmpdir.path) + .filter((file) => file.endsWith('.heapsnapshot')); + const risky = [...stderr.matchAll( + /Not generating snapshots because it's too risky/g)].length; + assert(list.length + risky > 0 && list.length <= 1, + `Generated ${list.length} snapshots ` + + `and ${risky} was too risky`); +} + +{ + console.log('\nTesting limit = 1'); + tmpdir.refresh(); + const child = spawnSync(process.execPath, [ + '--trace-gc', + '--max-old-space-size=50', + fixtures.path('workload', 'grow-and-set-near-heap-limit.js'), + ], { + cwd: tmpdir.path, + env: { + ...env, + limit: 1, + }, + }); + console.log(child.stdout.toString()); + const stderr = child.stderr.toString(); + console.log(stderr); + assert(common.nodeProcessAborted(child.status, child.signal), + 'process should have aborted, but did not'); + const list = fs.readdirSync(tmpdir.path) + .filter((file) => file.endsWith('.heapsnapshot')); + const risky = [...stderr.matchAll( + /Not generating snapshots because it's too risky/g)].length; + assert(list.length + risky > 0 && list.length <= 1, + `Generated ${list.length} snapshots ` + + `and ${risky} was too risky`); +} + +{ + console.log('\nTesting limit = 3'); + tmpdir.refresh(); + const child = spawnSync(process.execPath, [ + '--trace-gc', + '--max-old-space-size=50', + fixtures.path('workload', 'grow-and-set-near-heap-limit.js'), + ], { + cwd: tmpdir.path, + env: { + ...env, + limit: 3, + }, + }); + console.log(child.stdout.toString()); + const stderr = child.stderr.toString(); + console.log(stderr); + assert(common.nodeProcessAborted(child.status, child.signal), + 'process should have aborted, but did not'); + const list = fs.readdirSync(tmpdir.path) + .filter((file) => file.endsWith('.heapsnapshot')); + const risky = [...stderr.matchAll( + /Not generating snapshots because it's too risky/g)].length; + assert(list.length + risky > 0 && list.length <= 3, + `Generated ${list.length} snapshots ` + + `and ${risky} was too risky`); +} diff --git a/test/pummel/test-heapsnapshot-near-heap-limit.js b/test/pummel/test-heapsnapshot-near-heap-limit.js index 1af4e61e08c028..7677fe64ad854a 100644 --- a/test/pummel/test-heapsnapshot-near-heap-limit.js +++ b/test/pummel/test-heapsnapshot-near-heap-limit.js @@ -71,7 +71,7 @@ const env = { .filter((file) => file.endsWith('.heapsnapshot')); const risky = [...stderr.matchAll( /Not generating snapshots because it's too risky/g)].length; - assert(list.length + risky > 0 && list.length <= 3, + assert(list.length + risky > 0 && list.length <= 1, `Generated ${list.length} snapshots ` + `and ${risky} was too risky`); }