Skip to content

Commit

Permalink
src: unify Linux and FreeBSD large pages implem
Browse files Browse the repository at this point in the history 
dl_iterate_phdr(3) is also available for FreeBSD. This change adds the
same trimming code for the start and end of the .text section as on
Linux, making it work on FreeBSD, and removing the need for the
additional FreeBSD-specific check.

Manually tested on
  * https://www.osboxes.org/freebsd/#freebsd-12-1-vbox
  * https://www.osboxes.org/freebsd/#freebsd-11-vbox
  * test-digitalocean-freebsd11-x64-2
  • Loading branch information
Gabriel Schulhof committed Mar 28, 2020
1 parent fa3fd78 commit 5457085
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 84 deletions.
4 changes: 2 additions & 2 deletions node.gyp
View file
Expand Up @@ -323,7 +323,7 @@
'target_name': 'node_text_start',
'type': 'none',
'conditions': [
[ 'OS=="linux" and '
[ 'OS in "linux freebsd" and '
'target_arch=="x64"', {
'type': 'static_library',
'sources': [
Expand Down Expand Up @@ -512,7 +512,7 @@
'src/node_snapshot_stub.cc'
],
}],
[ 'OS=="linux" and '
[ 'OS in "linux freebsd" and '
'target_arch=="x64"', {
'dependencies': [ 'node_text_start' ],
'ldflags+': [
Expand Down
120 changes: 38 additions & 82 deletions src/large_pages/node_large_page.cc
View file
Expand Up @@ -30,12 +30,16 @@
#include "util.h"
#include "uv.h"

#if defined(__linux__) || defined(__FreeBSD__)
#include <string.h>
#if defined(__linux__)
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#endif // ifndef _GNU_SOURCE
#endif // defined(__linux__)
#include <link.h>
#endif
#endif // defined(__linux__) || defined(__FreeBSD__)

#include <sys/types.h>
#include <sys/mman.h>
#if defined(__FreeBSD__)
Expand Down Expand Up @@ -73,15 +77,15 @@
// Use madvise with MADV_HUGEPAGE to use Anonymous 2M Pages
// If successful copy the code there and unmap the original region.

#if defined(__linux__)
#if defined(__linux__) || defined(__FreeBSD__)
extern "C" {
// This symbol must be declared weak because this file becomes part of all
// Node.js targets (like node_mksnapshot, node_mkcodecache, and cctest) and
// those files do not supply the symbol.
extern char __attribute__((weak)) __node_text_start;
extern char __start_lpstub;
} // extern "C"
#endif // defined(__linux__)
#endif // defined(__linux__) || defined(__FreeBSD__)

#endif // defined(NODE_ENABLE_LARGE_CODE_PAGES) && NODE_ENABLE_LARGE_CODE_PAGES
namespace node {
Expand Down Expand Up @@ -121,19 +125,26 @@ inline uintptr_t hugepage_align_down(uintptr_t addr) {
return ((addr) & ~((hps) - 1));
}

#if defined(__linux__) || defined(__FreeBSD__)
#if defined(__FreeBSD__)
#ifndef ElfW
#define ElfW(name) Elf_##name
#endif // ifndef ElfW
#endif // defined(__FreeBSD__)

struct dl_iterate_params {
uintptr_t start;
uintptr_t end;
uintptr_t reference_sym;
std::string exename;
};

#if defined(__linux__)
int FindMapping(struct dl_phdr_info* info, size_t, void* data) {
if (info->dlpi_name[0] == 0) {
auto dl_params = static_cast<dl_iterate_params*>(data);
if (dl_params->exename == std::string(info->dlpi_name)) {
for (int idx = 0; idx < info->dlpi_phnum; idx++) {
const ElfW(Phdr)* phdr = &info->dlpi_phdr[idx];
if (phdr->p_type == PT_LOAD && (phdr->p_flags & PF_X)) {
auto dl_params = static_cast<dl_iterate_params*>(data);
uintptr_t start = info->dlpi_addr + phdr->p_vaddr;
uintptr_t end = start + phdr->p_memsz;

Expand All @@ -148,17 +159,30 @@ int FindMapping(struct dl_phdr_info* info, size_t, void* data) {
}
return 0;
}
#endif // defined(__linux__)
#endif // defined(__linux__) || defined(__FreeBSD__)

struct text_region FindNodeTextRegion() {
struct text_region nregion;
nregion.found_text_region = false;
#if defined(__linux__)
#if defined(__linux__) || defined(__FreeBSD__)
dl_iterate_params dl_params = {
0, 0, reinterpret_cast<uintptr_t>(&__node_text_start)
0, 0, reinterpret_cast<uintptr_t>(&__node_text_start), ""
};
uintptr_t lpstub_start = reinterpret_cast<uintptr_t>(&__start_lpstub);

#if defined(__FreeBSD__)
// On FreeBSD we need the name of the binary, because `dl_iterate_phdr` does
// not pass in an empty string as the `dlpi_name` of the binary but rather its
// absolute path.
{
char selfexe[PATH_MAX];
size_t count = sizeof(selfexe);
if (uv_exepath(selfexe, &count))
return nregion;
dl_params.exename = std::string(selfexe, count);
}
#endif // defined(__FreeBSD__)

if (dl_iterate_phdr(FindMapping, &dl_params) == 1) {
Debug("Hugepages info: start: %p - sym: %p - end: %p\n",
reinterpret_cast<void*>(dl_params.start),
Expand Down Expand Up @@ -187,62 +211,6 @@ struct text_region FindNodeTextRegion() {
}
}
}
#elif defined(__FreeBSD__)
std::string exename;
{
char selfexe[PATH_MAX];
size_t count = sizeof(selfexe);
if (uv_exepath(selfexe, &count))
return nregion;

exename = std::string(selfexe, count);
}

size_t numpg;
int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
const size_t miblen = arraysize(mib);
if (sysctl(mib, miblen, nullptr, &numpg, nullptr, 0) == -1) {
return nregion;
}

// Enough for struct kinfo_vmentry.
numpg = numpg * 4 / 3;
auto alg = std::vector<char>(numpg);

if (sysctl(mib, miblen, alg.data(), &numpg, nullptr, 0) == -1) {
return nregion;
}

char* start = alg.data();
char* end = start + numpg;

while (start < end) {
kinfo_vmentry* entry = reinterpret_cast<kinfo_vmentry*>(start);
const size_t cursz = entry->kve_structsize;
if (cursz == 0) {
break;
}

if (entry->kve_path[0] == '\0') {
continue;
}
bool excmapping = ((entry->kve_protection & KVME_PROT_READ) &&
(entry->kve_protection & KVME_PROT_EXEC));

if (!strcmp(exename.c_str(), entry->kve_path) && excmapping) {
char* estart =
reinterpret_cast<char*>(hugepage_align_up(entry->kve_start));
char* eend =
reinterpret_cast<char*>(hugepage_align_down(entry->kve_end));
size_t size = eend - estart;
nregion.found_text_region = true;
nregion.from = estart;
nregion.to = eend;
nregion.total_hugepages = size / hps;
break;
}
start += cursz;
}
#elif defined(__APPLE__)
struct vm_region_submap_info_64 map;
mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
Expand Down Expand Up @@ -349,13 +317,11 @@ MoveTextRegionToLargePages(const text_region& r) {
PrintSystemError(errno);
});

#if !defined (__FreeBSD__)
// Allocate temporary region and back up the code we will re-map.
nmem = mmap(nullptr, size,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (nmem == MAP_FAILED) goto fail;
memcpy(nmem, r.from, size);
#endif

#if defined(__linux__)
// We already know the original page is r-xp
Expand All @@ -365,15 +331,13 @@ MoveTextRegionToLargePages(const text_region& r) {
tmem = mmap(start, size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1 , 0);
if (tmem == MAP_FAILED) goto fail;
if (madvise(tmem, size, 14 /* MADV_HUGEPAGE */) == -1) goto fail;
memcpy(start, nmem, size);
if (tmem != MAP_FAILED)
if (madvise(tmem, size, 14 /* MADV_HUGEPAGE */) == -1) goto fail;
#elif defined(__FreeBSD__)
tmem = mmap(start, size,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED |
MAP_ALIGNED_SUPER, -1 , 0);
if (tmem == MAP_FAILED) goto fail;
#elif defined(__APPLE__)
// There is not enough room to reserve the mapping close
// to the region address so we content to give a hint
Expand All @@ -384,12 +348,9 @@ MoveTextRegionToLargePages(const text_region& r) {
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_PRIVATE | MAP_ANONYMOUS,
VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
if (tmem == MAP_FAILED) goto fail;
memcpy(tmem, nmem, size);
if (mprotect(start, size, PROT_READ | PROT_WRITE | PROT_EXEC) == -1)
goto fail;
memcpy(start, tmem, size);
#endif
if (tmem == MAP_FAILED) goto fail;
memcpy(start, nmem, size);

if (mprotect(start, size, PROT_READ | PROT_EXEC) == -1) goto fail;
// We need not `munmap(tmem, size)` in the above `OnScopeLeave` on success.
Expand Down Expand Up @@ -420,11 +381,6 @@ int MapStaticCodeToLargePages() {
if (r.found_text_region == false)
return ENOENT;

#if defined(__FreeBSD__)
if (r.from < reinterpret_cast<void*>(&MoveTextRegionToLargePages))
return -1;
#endif

return MoveTextRegionToLargePages(r);
#else
return ENOTSUP;
Expand Down

0 comments on commit 5457085

Please sign in to comment.