From ae1fa9849627e16ab7507a705f787fc5738639fb Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Thu, 4 Feb 2021 13:25:48 +0100 Subject: [PATCH] deps: cherry-pick f4376ec801e1ded from V8 upstream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit message: [heap] Make maximum regular code object size a runtime value. Executable V8 pages include 3 reserved OS pages: one for the writable header and two as guards. On systems with 64k OS pages, the amount of allocatable space left for objects can then be quite smaller than the page size, only 64k for each 256k page. This means regular code objects cannot be larger than 64k, while the maximum regular object size is fixed to 128k, half of the page size. As a result code object never reach this limit and we can end up filling regular pages with few large code objects. To fix this, we change the maximum code object size to be runtime value, set to half of the allocatable space per page. On systems with 64k OS pages, the limit will be 32k. Alternatively, we could increase the V8 page size to 512k on Arm64 linux so we wouldn't waste code space. However, systems with 4k OS pages are more common, and those with 64k pages tend to have more memory available so we should be able to live with it. Bug: v8:10808 Change-Id: I5d807e7a3df89f1e9c648899e9ba2f8e2648264c Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2460809 Reviewed-by: Igor Sheludko Reviewed-by: Georg Neis Reviewed-by: Ulan Degenbaev Commit-Queue: Pierre Langlois Cr-Commit-Position: refs/heads/master@{#70569} PR-URL: https://github.com/nodejs/node/pull/37225 Refs: https://github.com/nodejs/help/issues/3202 Reviewed-By: Michael Dawson Reviewed-By: Stewart X Addison Reviewed-By: Juan José Arboleda Reviewed-By: James M Snell --- deps/v8/src/base/build_config.h | 4 + deps/v8/src/compiler/allocation-builder.h | 2 +- deps/v8/src/compiler/memory-lowering.cc | 4 + deps/v8/src/diagnostics/objects-debug.cc | 3 +- deps/v8/src/heap/factory-base.cc | 3 +- deps/v8/src/heap/factory.cc | 3 +- deps/v8/src/heap/heap-inl.h | 6 +- deps/v8/src/heap/heap.cc | 8 ++ deps/v8/src/heap/heap.h | 16 +++- deps/v8/src/heap/memory-chunk.cc | 7 +- deps/v8/src/heap/memory-chunk.h | 2 + deps/v8/src/heap/spaces.h | 6 +- deps/v8/src/utils/allocation.cc | 3 +- deps/v8/test/cctest/heap/heap-tester.h | 1 + deps/v8/test/cctest/heap/test-heap.cc | 85 ++++++++++++++++++- deps/v8/test/cctest/test-code-pages.cc | 6 +- deps/v8/test/cctest/test-factory.cc | 3 +- .../test/cctest/test-unwinder-code-pages.cc | 2 +- 18 files changed, 143 insertions(+), 21 deletions(-) diff --git a/deps/v8/src/base/build_config.h b/deps/v8/src/base/build_config.h index 8d142c456c9cc1..327f9ab3060e3d 100644 --- a/deps/v8/src/base/build_config.h +++ b/deps/v8/src/base/build_config.h @@ -205,6 +205,10 @@ // PPC has large (64KB) physical pages. const int kPageSizeBits = 19; #else +// Arm64 supports up to 64k OS pages on Linux, however 4k pages are more common +// so we keep the V8 page size at 256k. Nonetheless, we need to make sure we +// don't decrease it further in the future due to reserving 3 OS pages for every +// executable V8 page. const int kPageSizeBits = 18; #endif diff --git a/deps/v8/src/compiler/allocation-builder.h b/deps/v8/src/compiler/allocation-builder.h index 040dd014051270..3a8dbdfff71804 100644 --- a/deps/v8/src/compiler/allocation-builder.h +++ b/deps/v8/src/compiler/allocation-builder.h @@ -27,7 +27,7 @@ class AllocationBuilder final { // Primitive allocation of static size. void Allocate(int size, AllocationType allocation = AllocationType::kYoung, Type type = Type::Any()) { - DCHECK_LE(size, kMaxRegularHeapObjectSize); + DCHECK_LE(size, Heap::MaxRegularHeapObjectSize(allocation)); effect_ = graph()->NewNode( common()->BeginRegion(RegionObservability::kNotObservable), effect_); allocation_ = diff --git a/deps/v8/src/compiler/memory-lowering.cc b/deps/v8/src/compiler/memory-lowering.cc index 8c230b6efd2fdd..d9ab0b8174dd14 100644 --- a/deps/v8/src/compiler/memory-lowering.cc +++ b/deps/v8/src/compiler/memory-lowering.cc @@ -98,6 +98,10 @@ Reduction MemoryLowering::ReduceAllocateRaw( DCHECK_EQ(IrOpcode::kAllocateRaw, node->opcode()); DCHECK_IMPLIES(allocation_folding_ == AllocationFolding::kDoAllocationFolding, state_ptr != nullptr); + // Code objects may have a maximum size smaller than kMaxHeapObjectSize due to + // guard pages. If we need to support allocating code here we would need to + // call MemoryChunkLayout::MaxRegularCodeObjectSize() at runtime. + DCHECK_NE(allocation_type, AllocationType::kCode); Node* value; Node* size = node->InputAt(0); Node* effect = node->InputAt(1); diff --git a/deps/v8/src/diagnostics/objects-debug.cc b/deps/v8/src/diagnostics/objects-debug.cc index 32caba2da849b0..6df2aec1ed3dba 100644 --- a/deps/v8/src/diagnostics/objects-debug.cc +++ b/deps/v8/src/diagnostics/objects-debug.cc @@ -955,7 +955,8 @@ void Code::CodeVerify(Isolate* isolate) { // everything is set up. // CHECK_EQ(ReadOnlyHeap::Contains(*this), !IsExecutable()); relocation_info().ObjectVerify(isolate); - CHECK(Code::SizeFor(body_size()) <= kMaxRegularHeapObjectSize || + CHECK(Code::SizeFor(body_size()) <= + MemoryChunkLayout::MaxRegularCodeObjectSize() || isolate->heap()->InSpace(*this, CODE_LO_SPACE)); Address last_gc_pc = kNullAddress; diff --git a/deps/v8/src/heap/factory-base.cc b/deps/v8/src/heap/factory-base.cc index 028949e861d1cb..9113eaee133519 100644 --- a/deps/v8/src/heap/factory-base.cc +++ b/deps/v8/src/heap/factory-base.cc @@ -721,7 +721,8 @@ template HeapObject FactoryBase::AllocateRawArray(int size, AllocationType allocation) { HeapObject result = AllocateRaw(size, allocation); - if (size > kMaxRegularHeapObjectSize && FLAG_use_marking_progress_bar) { + if ((size > Heap::MaxRegularHeapObjectSize(allocation)) && + FLAG_use_marking_progress_bar) { MemoryChunk* chunk = MemoryChunk::FromHeapObject(result); chunk->SetFlag(MemoryChunk::HAS_PROGRESS_BAR); } diff --git a/deps/v8/src/heap/factory.cc b/deps/v8/src/heap/factory.cc index 7e4f897fee4ffc..a87c4995cf2144 100644 --- a/deps/v8/src/heap/factory.cc +++ b/deps/v8/src/heap/factory.cc @@ -346,7 +346,8 @@ MaybeHandle Factory::TryNewFixedArray( AllocationResult allocation = heap->AllocateRaw(size, allocation_type); HeapObject result; if (!allocation.To(&result)) return MaybeHandle(); - if (size > kMaxRegularHeapObjectSize && FLAG_use_marking_progress_bar) { + if ((size > Heap::MaxRegularHeapObjectSize(allocation_type)) && + FLAG_use_marking_progress_bar) { MemoryChunk* chunk = MemoryChunk::FromHeapObject(result); chunk->SetFlag(MemoryChunk::HAS_PROGRESS_BAR); } diff --git a/deps/v8/src/heap/heap-inl.h b/deps/v8/src/heap/heap-inl.h index 39f5ec6c66e7f5..cb7d05c3b0b596 100644 --- a/deps/v8/src/heap/heap-inl.h +++ b/deps/v8/src/heap/heap-inl.h @@ -192,7 +192,9 @@ AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type, IncrementObjectCounters(); #endif - bool large_object = size_in_bytes > kMaxRegularHeapObjectSize; + size_t large_object_threshold = MaxRegularHeapObjectSize(type); + bool large_object = + static_cast(size_in_bytes) > large_object_threshold; HeapObject object; AllocationResult allocation; @@ -279,7 +281,7 @@ HeapObject Heap::AllocateRawWith(int size, AllocationType allocation, Address* limit = heap->NewSpaceAllocationLimitAddress(); if (allocation == AllocationType::kYoung && alignment == AllocationAlignment::kWordAligned && - size <= kMaxRegularHeapObjectSize && + size <= MaxRegularHeapObjectSize(allocation) && (*limit - *top >= static_cast(size)) && V8_LIKELY(!FLAG_single_generation && FLAG_inline_new && FLAG_gc_interval == 0)) { diff --git a/deps/v8/src/heap/heap.cc b/deps/v8/src/heap/heap.cc index d035fece83b799..5d5eaae0683b74 100644 --- a/deps/v8/src/heap/heap.cc +++ b/deps/v8/src/heap/heap.cc @@ -4963,6 +4963,14 @@ bool Heap::AllocationLimitOvershotByLargeMargin() { return v8_overshoot >= v8_margin || global_overshoot >= global_margin; } +// static +int Heap::MaxRegularHeapObjectSize(AllocationType allocation) { + if (allocation == AllocationType::kCode) { + return MemoryChunkLayout::MaxRegularCodeObjectSize(); + } + return kMaxRegularHeapObjectSize; +} + bool Heap::ShouldOptimizeForLoadTime() { return isolate()->rail_mode() == PERFORMANCE_LOAD && !AllocationLimitOvershotByLargeMargin() && diff --git a/deps/v8/src/heap/heap.h b/deps/v8/src/heap/heap.h index 888d174c02fb4c..b2105a96badc2f 100644 --- a/deps/v8/src/heap/heap.h +++ b/deps/v8/src/heap/heap.h @@ -481,7 +481,7 @@ class Heap { bool IsImmovable(HeapObject object); - static bool IsLargeObject(HeapObject object); + V8_EXPORT_PRIVATE static bool IsLargeObject(HeapObject object); // This method supports the deserialization allocator. All allocations // are word-aligned. The method should never fail to allocate since the @@ -1316,6 +1316,14 @@ class Heap { // more eager to finalize incremental marking. bool AllocationLimitOvershotByLargeMargin(); + // Return the maximum size objects can be before having to allocate them as + // large objects. This takes into account allocating in the code space for + // which the size of the allocatable space per V8 page may depend on the OS + // page size at runtime. You may use kMaxRegularHeapObjectSize as a constant + // instead if you know the allocation isn't in the code spaces. + V8_EXPORT_PRIVATE static int MaxRegularHeapObjectSize( + AllocationType allocation); + // =========================================================================== // Prologue/epilogue callback methods.======================================== // =========================================================================== @@ -1404,8 +1412,10 @@ class Heap { // Heap object allocation tracking. ========================================== // =========================================================================== - void AddHeapObjectAllocationTracker(HeapObjectAllocationTracker* tracker); - void RemoveHeapObjectAllocationTracker(HeapObjectAllocationTracker* tracker); + V8_EXPORT_PRIVATE void AddHeapObjectAllocationTracker( + HeapObjectAllocationTracker* tracker); + V8_EXPORT_PRIVATE void RemoveHeapObjectAllocationTracker( + HeapObjectAllocationTracker* tracker); bool has_heap_object_allocation_tracker() const { return !allocation_trackers_.empty(); } diff --git a/deps/v8/src/heap/memory-chunk.cc b/deps/v8/src/heap/memory-chunk.cc index 865e6f1a72b23b..854699b862137b 100644 --- a/deps/v8/src/heap/memory-chunk.cc +++ b/deps/v8/src/heap/memory-chunk.cc @@ -47,7 +47,6 @@ intptr_t MemoryChunkLayout::ObjectEndOffsetInCodePage() { size_t MemoryChunkLayout::AllocatableMemoryInCodePage() { size_t memory = ObjectEndOffsetInCodePage() - ObjectStartOffsetInCodePage(); - DCHECK_LE(kMaxRegularHeapObjectSize, memory); return memory; } @@ -77,6 +76,12 @@ size_t MemoryChunkLayout::AllocatableMemoryInMemoryChunk( return AllocatableMemoryInDataPage(); } +int MemoryChunkLayout::MaxRegularCodeObjectSize() { + int size = static_cast(AllocatableMemoryInCodePage() / 2); + DCHECK_LE(size, kMaxRegularHeapObjectSize); + return size; +} + #ifdef THREAD_SANITIZER void MemoryChunk::SynchronizedHeapLoad() { CHECK(reinterpret_cast(base::Acquire_Load( diff --git a/deps/v8/src/heap/memory-chunk.h b/deps/v8/src/heap/memory-chunk.h index 4381a229ab2517..58fdbefa1742d5 100644 --- a/deps/v8/src/heap/memory-chunk.h +++ b/deps/v8/src/heap/memory-chunk.h @@ -32,6 +32,8 @@ class V8_EXPORT_PRIVATE MemoryChunkLayout { static size_t AllocatableMemoryInDataPage(); static size_t ObjectStartOffsetInMemoryChunk(AllocationSpace space); static size_t AllocatableMemoryInMemoryChunk(AllocationSpace space); + + static int MaxRegularCodeObjectSize(); }; // MemoryChunk represents a memory region owned by a specific space. diff --git a/deps/v8/src/heap/spaces.h b/deps/v8/src/heap/spaces.h index 72ae96cadd2be0..50688dda2012d7 100644 --- a/deps/v8/src/heap/spaces.h +++ b/deps/v8/src/heap/spaces.h @@ -127,8 +127,10 @@ class Space; #define DCHECK_OBJECT_SIZE(size) \ DCHECK((0 < size) && (size <= kMaxRegularHeapObjectSize)) -#define DCHECK_CODEOBJECT_SIZE(size, code_space) \ - DCHECK((0 < size) && (size <= code_space->AreaSize())) +#define DCHECK_CODEOBJECT_SIZE(size, code_space) \ + DCHECK((0 < size) && \ + (size <= std::min(MemoryChunkLayout::MaxRegularCodeObjectSize(), \ + code_space->AreaSize()))) using FreeListCategoryType = int32_t; diff --git a/deps/v8/src/utils/allocation.cc b/deps/v8/src/utils/allocation.cc index 12dfaf95726f1e..cf820c37e73687 100644 --- a/deps/v8/src/utils/allocation.cc +++ b/deps/v8/src/utils/allocation.cc @@ -165,8 +165,7 @@ void* GetRandomMmapAddr() { void* AllocatePages(v8::PageAllocator* page_allocator, void* hint, size_t size, size_t alignment, PageAllocator::Permission access) { DCHECK_NOT_NULL(page_allocator); - DCHECK_EQ(hint, AlignedAddress(hint, alignment)); - DCHECK(IsAligned(size, page_allocator->AllocatePageSize())); + DCHECK(IsAligned(size, page_allocator->CommitPageSize())); if (FLAG_randomize_all_allocations) { hint = page_allocator->GetRandomMmapAddr(); } diff --git a/deps/v8/test/cctest/heap/heap-tester.h b/deps/v8/test/cctest/heap/heap-tester.h index 998e3ff011fa3e..1b001e99a4d679 100644 --- a/deps/v8/test/cctest/heap/heap-tester.h +++ b/deps/v8/test/cctest/heap/heap-tester.h @@ -34,6 +34,7 @@ V(InvalidatedSlotsSomeInvalidatedRanges) \ V(TestNewSpaceRefsInCopiedCode) \ V(GCFlags) \ + V(CodeLargeObjectSpace64k) \ V(MarkCompactCollector) \ V(MarkCompactEpochCounter) \ V(MemoryReducerActivationForSmallHeaps) \ diff --git a/deps/v8/test/cctest/heap/test-heap.cc b/deps/v8/test/cctest/heap/test-heap.cc index b9a4b2101cc775..9c349cd1c39c91 100644 --- a/deps/v8/test/cctest/heap/test-heap.cc +++ b/deps/v8/test/cctest/heap/test-heap.cc @@ -6404,7 +6404,7 @@ HEAP_TEST(Regress5831) { // Generate the code. Handle code = GenerateDummyImmovableCode(isolate); - CHECK_GE(i::kMaxRegularHeapObjectSize, code->Size()); + CHECK_GE(MemoryChunkLayout::MaxRegularCodeObjectSize(), code->Size()); CHECK(!heap->code_space()->first_page()->Contains(code->address())); // Ensure it's not in large object space. @@ -6889,7 +6889,7 @@ TEST(CodeObjectRegistry) { { // Ensure that both code objects end up on the same page. CHECK(HeapTester::CodeEnsureLinearAllocationArea( - heap, kMaxRegularHeapObjectSize)); + heap, MemoryChunkLayout::MaxRegularCodeObjectSize())); code1 = DummyOptimizedCode(isolate); Handle code2 = DummyOptimizedCode(isolate); code2_address = code2->address(); @@ -7002,6 +7002,87 @@ TEST(Regress978156) { marking_state->GreyToBlack(filler); } +class TestAllocationTracker : public HeapObjectAllocationTracker { + public: + explicit TestAllocationTracker(int expected_size) + : expected_size_(expected_size) {} + + void AllocationEvent(Address addr, int size) { + CHECK(expected_size_ == size); + address_ = addr; + } + + Address address() { return address_; } + + private: + int expected_size_; + Address address_; +}; + +UNINITIALIZED_HEAP_TEST(CodeLargeObjectSpace64k) { + // Simulate having a system with 64k OS pages. + i::FLAG_v8_os_page_size = 64; + + // Initialize the isolate manually to make sure --v8-os-page-size is taken + // into account. + v8::Isolate::CreateParams create_params; + create_params.array_buffer_allocator = CcTest::array_buffer_allocator(); + v8::Isolate* isolate = v8::Isolate::New(create_params); + + Heap* heap = reinterpret_cast(isolate)->heap(); + + // Allocate a regular code object. + { + int size_in_bytes = + MemoryChunkLayout::MaxRegularCodeObjectSize() - kTaggedSize; + TestAllocationTracker allocation_tracker{size_in_bytes}; + heap->AddHeapObjectAllocationTracker(&allocation_tracker); + + HeapObject obj; + { + AllocationResult allocation = heap->AllocateRaw( + size_in_bytes, AllocationType::kCode, AllocationOrigin::kRuntime, + AllocationAlignment::kCodeAligned); + CHECK(allocation.To(&obj)); + CHECK_EQ(allocation.ToObjectChecked().address(), + allocation_tracker.address()); + + heap->CreateFillerObjectAt(obj.address(), size_in_bytes, + ClearRecordedSlots::kNo); + } + + CHECK(!Heap::IsLargeObject(obj)); + heap->RemoveHeapObjectAllocationTracker(&allocation_tracker); + } + + // Allocate a large code object. + { + int size_in_bytes = + MemoryChunkLayout::MaxRegularCodeObjectSize() + kTaggedSize; + TestAllocationTracker allocation_tracker{size_in_bytes}; + heap->AddHeapObjectAllocationTracker(&allocation_tracker); + + HeapObject obj; + { + AllocationResult allocation = heap->AllocateRaw( + size_in_bytes, AllocationType::kCode, AllocationOrigin::kRuntime, + AllocationAlignment::kCodeAligned); + CHECK(allocation.To(&obj)); + CHECK_EQ(allocation.ToObjectChecked().address(), + allocation_tracker.address()); + + heap->CreateFillerObjectAt(obj.address(), size_in_bytes, + ClearRecordedSlots::kNo); + } + + CHECK(Heap::IsLargeObject(obj)); + heap->RemoveHeapObjectAllocationTracker(&allocation_tracker); + } + + isolate->Dispose(); +} + + } // namespace heap } // namespace internal } // namespace v8 diff --git a/deps/v8/test/cctest/test-code-pages.cc b/deps/v8/test/cctest/test-code-pages.cc index 7fafe626296ef6..9624a79504eacf 100644 --- a/deps/v8/test/cctest/test-code-pages.cc +++ b/deps/v8/test/cctest/test-code-pages.cc @@ -263,7 +263,7 @@ TEST(LargeCodeObject) { // Create a big function that ends up in CODE_LO_SPACE. const int instruction_size = Page::kPageSize + 1; - STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize); + CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize()); std::unique_ptr instructions(new byte[instruction_size]); CodeDesc desc; @@ -379,7 +379,7 @@ TEST(LargeCodeObjectWithSignalHandler) { // Create a big function that ends up in CODE_LO_SPACE. const int instruction_size = Page::kPageSize + 1; - STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize); + CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize()); std::unique_ptr instructions(new byte[instruction_size]); CodeDesc desc; @@ -455,7 +455,7 @@ TEST(Sorted) { // Create a big function that ends up in CODE_LO_SPACE. const int instruction_size = Page::kPageSize + 1; - STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize); + CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize()); std::unique_ptr instructions(new byte[instruction_size]); CodeDesc desc; diff --git a/deps/v8/test/cctest/test-factory.cc b/deps/v8/test/cctest/test-factory.cc index 98823f1ec9eaa5..859698f88650ca 100644 --- a/deps/v8/test/cctest/test-factory.cc +++ b/deps/v8/test/cctest/test-factory.cc @@ -60,7 +60,8 @@ TEST(Factory_CodeBuilder) { HandleScope scope(isolate); // Create a big function that ends up in CODE_LO_SPACE. - const int instruction_size = kMaxRegularHeapObjectSize + 1; + const int instruction_size = + MemoryChunkLayout::MaxRegularCodeObjectSize() + 1; std::unique_ptr instructions(new byte[instruction_size]); CodeDesc desc; diff --git a/deps/v8/test/cctest/test-unwinder-code-pages.cc b/deps/v8/test/cctest/test-unwinder-code-pages.cc index fc023e41455f0f..40001d51b1dc3e 100644 --- a/deps/v8/test/cctest/test-unwinder-code-pages.cc +++ b/deps/v8/test/cctest/test-unwinder-code-pages.cc @@ -564,7 +564,7 @@ TEST(PCIsInV8_LargeCodeObject_CodePagesAPI) { // Create a big function that ends up in CODE_LO_SPACE. const int instruction_size = Page::kPageSize + 1; - STATIC_ASSERT(instruction_size > kMaxRegularHeapObjectSize); + CHECK_GT(instruction_size, MemoryChunkLayout::MaxRegularCodeObjectSize()); std::unique_ptr instructions(new byte[instruction_size]); CodeDesc desc;