From 23f8f350228b2ad77a52f65bcf63188424daab6e Mon Sep 17 00:00:00 2001 From: Gabriel Schulhof Date: Thu, 26 Mar 2020 03:14:59 +0000 Subject: [PATCH] src: unify Linux and FreeBSD large pages implem dl_iterate_phdr(3) is also available for FreeBSD. This change adds the same trimming code for the start and end of the .text section as on Linux, making it work on FreeBSD, and removing the need for the additional FreeBSD-specific check. Manually tested on * https://www.osboxes.org/freebsd/#freebsd-12-1-vbox * https://www.osboxes.org/freebsd/#freebsd-11-vbox * test-digitalocean-freebsd11-x64-2 Signed-off-by: Gabriel Schulhof PR-URL: https://github.com/nodejs/node/pull/32534 Reviewed-By: Ben Noordhuis Reviewed-By: James M Snell Reviewed-By: David Carlier --- node.gyp | 4 +- src/large_pages/node_large_page.cc | 108 ++++++++++------------------- 2 files changed, 37 insertions(+), 75 deletions(-) diff --git a/node.gyp b/node.gyp index b6d8668ecf34d2..152ee52e46611b 100644 --- a/node.gyp +++ b/node.gyp @@ -321,7 +321,7 @@ 'target_name': 'node_text_start', 'type': 'none', 'conditions': [ - [ 'OS=="linux" and ' + [ 'OS in "linux freebsd" and ' 'target_arch=="x64"', { 'type': 'static_library', 'sources': [ @@ -510,7 +510,7 @@ 'src/node_snapshot_stub.cc' ], }], - [ 'OS=="linux" and ' + [ 'OS in "linux freebsd" and ' 'target_arch=="x64"', { 'dependencies': [ 'node_text_start' ], 'ldflags+': [ diff --git a/src/large_pages/node_large_page.cc b/src/large_pages/node_large_page.cc index 7c747047b6ab9a..d0200937b727e6 100644 --- a/src/large_pages/node_large_page.cc +++ b/src/large_pages/node_large_page.cc @@ -30,12 +30,16 @@ #include "util.h" #include "uv.h" +#if defined(__linux__) || defined(__FreeBSD__) +#include #if defined(__linux__) #ifndef _GNU_SOURCE #define _GNU_SOURCE -#endif +#endif // ifndef _GNU_SOURCE +#endif // defined(__linux__) #include -#endif +#endif // defined(__linux__) || defined(__FreeBSD__) + #include #include #if defined(__FreeBSD__) @@ -73,7 +77,7 @@ // Use madvise with MADV_HUGEPAGE to use Anonymous 2M Pages // If successful copy the code there and unmap the original region. -#if defined(__linux__) +#if defined(__linux__) || defined(__FreeBSD__) extern "C" { // This symbol must be declared weak because this file becomes part of all // Node.js targets (like node_mksnapshot, node_mkcodecache, and cctest) and @@ -81,7 +85,7 @@ extern "C" { extern char __attribute__((weak)) __node_text_start; extern char __start_lpstub; } // extern "C" -#endif // defined(__linux__) +#endif // defined(__linux__) || defined(__FreeBSD__) #endif // defined(NODE_ENABLE_LARGE_CODE_PAGES) && NODE_ENABLE_LARGE_CODE_PAGES namespace node { @@ -121,19 +125,26 @@ inline uintptr_t hugepage_align_down(uintptr_t addr) { return ((addr) & ~((hps) - 1)); } +#if defined(__linux__) || defined(__FreeBSD__) +#if defined(__FreeBSD__) +#ifndef ElfW +#define ElfW(name) Elf_##name +#endif // ifndef ElfW +#endif // defined(__FreeBSD__) + struct dl_iterate_params { uintptr_t start; uintptr_t end; uintptr_t reference_sym; + std::string exename; }; -#if defined(__linux__) int FindMapping(struct dl_phdr_info* info, size_t, void* data) { - if (info->dlpi_name[0] == 0) { + auto dl_params = static_cast(data); + if (dl_params->exename == std::string(info->dlpi_name)) { for (int idx = 0; idx < info->dlpi_phnum; idx++) { const ElfW(Phdr)* phdr = &info->dlpi_phdr[idx]; if (phdr->p_type == PT_LOAD && (phdr->p_flags & PF_X)) { - auto dl_params = static_cast(data); uintptr_t start = info->dlpi_addr + phdr->p_vaddr; uintptr_t end = start + phdr->p_memsz; @@ -148,17 +159,30 @@ int FindMapping(struct dl_phdr_info* info, size_t, void* data) { } return 0; } -#endif // defined(__linux__) +#endif // defined(__linux__) || defined(__FreeBSD__) struct text_region FindNodeTextRegion() { struct text_region nregion; nregion.found_text_region = false; -#if defined(__linux__) +#if defined(__linux__) || defined(__FreeBSD__) dl_iterate_params dl_params = { - 0, 0, reinterpret_cast(&__node_text_start) + 0, 0, reinterpret_cast(&__node_text_start), "" }; uintptr_t lpstub_start = reinterpret_cast(&__start_lpstub); +#if defined(__FreeBSD__) + // On FreeBSD we need the name of the binary, because `dl_iterate_phdr` does + // not pass in an empty string as the `dlpi_name` of the binary but rather its + // absolute path. + { + char selfexe[PATH_MAX]; + size_t count = sizeof(selfexe); + if (uv_exepath(selfexe, &count)) + return nregion; + dl_params.exename = std::string(selfexe, count); + } +#endif // defined(__FreeBSD__) + if (dl_iterate_phdr(FindMapping, &dl_params) == 1) { Debug("Hugepages info: start: %p - sym: %p - end: %p\n", reinterpret_cast(dl_params.start), @@ -187,62 +211,6 @@ struct text_region FindNodeTextRegion() { } } } -#elif defined(__FreeBSD__) - std::string exename; - { - char selfexe[PATH_MAX]; - size_t count = sizeof(selfexe); - if (uv_exepath(selfexe, &count)) - return nregion; - - exename = std::string(selfexe, count); - } - - size_t numpg; - int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()}; - const size_t miblen = arraysize(mib); - if (sysctl(mib, miblen, nullptr, &numpg, nullptr, 0) == -1) { - return nregion; - } - - // Enough for struct kinfo_vmentry. - numpg = numpg * 4 / 3; - auto alg = std::vector(numpg); - - if (sysctl(mib, miblen, alg.data(), &numpg, nullptr, 0) == -1) { - return nregion; - } - - char* start = alg.data(); - char* end = start + numpg; - - while (start < end) { - kinfo_vmentry* entry = reinterpret_cast(start); - const size_t cursz = entry->kve_structsize; - if (cursz == 0) { - break; - } - - if (entry->kve_path[0] == '\0') { - continue; - } - bool excmapping = ((entry->kve_protection & KVME_PROT_READ) && - (entry->kve_protection & KVME_PROT_EXEC)); - - if (!strcmp(exename.c_str(), entry->kve_path) && excmapping) { - char* estart = - reinterpret_cast(hugepage_align_up(entry->kve_start)); - char* eend = - reinterpret_cast(hugepage_align_down(entry->kve_end)); - size_t size = eend - estart; - nregion.found_text_region = true; - nregion.from = estart; - nregion.to = eend; - nregion.total_hugepages = size / hps; - break; - } - start += cursz; - } #elif defined(__APPLE__) struct vm_region_submap_info_64 map; mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64; @@ -349,13 +317,11 @@ MoveTextRegionToLargePages(const text_region& r) { PrintSystemError(errno); }); -#if !defined (__FreeBSD__) // Allocate temporary region and back up the code we will re-map. nmem = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (nmem == MAP_FAILED) goto fail; memcpy(nmem, r.from, size); -#endif #if defined(__linux__) // We already know the original page is r-xp @@ -374,6 +340,7 @@ MoveTextRegionToLargePages(const text_region& r) { MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED | MAP_ALIGNED_SUPER, -1 , 0); if (tmem == MAP_FAILED) goto fail; + memcpy(start, nmem, size); #elif defined(__APPLE__) // There is not enough room to reserve the mapping close // to the region address so we content to give a hint @@ -420,11 +387,6 @@ int MapStaticCodeToLargePages() { if (r.found_text_region == false) return ENOENT; -#if defined(__FreeBSD__) - if (r.from < reinterpret_cast(&MoveTextRegionToLargePages)) - return -1; -#endif - return MoveTextRegionToLargePages(r); #else return ENOTSUP;