[llvm] Simple hugify fixes and refactoring (PR #202609)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 06:22:21 PDT 2026
https://github.com/DmitriiMartynov created https://github.com/llvm/llvm-project/pull/202609
None
>From 901ecaf1037c78aa42ed80126d83ce30d43fa9c5 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 5 May 2026 09:25:41 -0400
Subject: [PATCH 1/6] [BOLT] hugify: macro constants instead of hardcoded
values
Almost all values are already defined in the bolt/runtime/common.h file
---
bolt/runtime/common.h | 5 +++++
bolt/runtime/hugify.cpp | 25 +++++++++++--------------
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 8689bc8b72041..8547cad2c7a6d 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -64,6 +64,11 @@ typedef int int32_t;
#define MAP_FAILED ((void *)-1)
+#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
+
+/* set the state of the "THP disable" flags for the calling thread */
+#define PR_SET_THP_DISABLE 41
+
#define SEEK_SET 0 /* Seek from beginning of file. */
#define SEEK_CUR 1 /* Seek from current position. */
#define SEEK_END 2 /* Seek from end of file. */
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index de896307f24fc..34edf3cb1d03b 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -64,8 +64,8 @@ static void getKernelVersion(uint32_t *Val) {
static bool hasPagecacheTHPSupport() {
char Buf[64];
- int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
- 0 /* O_RDONLY */, 0);
+ const int FD =
+ __open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY, 0);
if (FD < 0)
return false;
@@ -98,11 +98,10 @@ static bool hasPagecacheTHPSupport() {
static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
const size_t Size = To - From;
- uint8_t *Mem = reinterpret_cast<uint8_t *>(
- __mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
- 0x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
+ void *Mem = __mmap(0, Size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (Mem == ((void *)-1) /* MAP_FAILED */) {
+ if (Mem == MAP_FAILED) {
char Msg[] = "[hugify] could not allocate memory for text move\n";
reportError(Msg, sizeof(Msg));
}
@@ -114,19 +113,17 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
// Copy the hot code to a temporary location.
memcpy(Mem, From, Size);
- __prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
+ __prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- if (__mmap(reinterpret_cast<uint64_t>(From), Size,
- 0x3 /* PROT_READ | PROT_WRITE */,
- 0x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
- 0) == ((void *)-1) /*MAP_FAILED*/) {
+ if (__mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0) == MAP_FAILED) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
reportError(Msg, sizeof(Msg));
}
// Mark the hot code page to be huge page.
- if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
+ if (__madvise(From, Size, MADV_HUGEPAGE) == -1) {
char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
@@ -135,7 +132,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
memcpy(From, Mem, Size);
// Change permission back to read-only, ignore failure
- __mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
+ __mprotect(From, Size, PROT_READ | PROT_EXEC);
__munmap(Mem, Size);
}
@@ -161,7 +158,7 @@ extern "C" void __bolt_hugify_self_impl() {
return;
}
- if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) == -1) {
char Msg[] = "[hugify] failed to allocate large page\n";
// TODO: allow user to control the failure behavior.
reportError(Msg, sizeof(Msg));
>From 68655caad0c492fc9903314c2d6d792dcef30281 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 5 May 2026 07:42:19 -0400
Subject: [PATCH 2/6] [BOLT] runtime: check __mmap return value correctly
The __mmap function is a system call wrapper.
The return value of the wrapper is either a valid address or
an error (a negative value), not MAP_FAILED.
---
bolt/runtime/common.h | 9 +++++++--
bolt/runtime/hugify.cpp | 8 +++++---
bolt/runtime/instr.cpp | 9 ++++-----
3 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 8547cad2c7a6d..0014897ade6f9 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -62,8 +62,6 @@ typedef int int32_t;
#define MAP_ANONYMOUS 0x20
#endif
-#define MAP_FAILED ((void *)-1)
-
#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
/* set the state of the "THP disable" flags for the calling thread */
@@ -368,4 +366,11 @@ inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
return (Value + Align - 1) / Align * Align;
}
+// See include/linux/err.h file in the Linux kernel
+constexpr intptr_t MaxErrno = 4095;
+inline bool isErrValue(const void *Val) {
+ const intptr_t PtrVal = reinterpret_cast<intptr_t>(Val);
+ return PtrVal >= -MaxErrno && PtrVal <= -1;
+}
+
} // anonymous namespace
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index 34edf3cb1d03b..66c376305e689 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -101,7 +101,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
void *Mem = __mmap(0, Size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (Mem == MAP_FAILED) {
+ if (isErrValue(Mem)) {
char Msg[] = "[hugify] could not allocate memory for text move\n";
reportError(Msg, sizeof(Msg));
}
@@ -115,8 +115,10 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
__prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- if (__mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0) == MAP_FAILED) {
+ const void *Addr =
+ __mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ if (isErrValue(Addr)) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
reportError(Msg, sizeof(Msg));
diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp
index 3470e391224ba..e8b4542e4d2e3 100644
--- a/bolt/runtime/instr.cpp
+++ b/bolt/runtime/instr.cpp
@@ -141,8 +141,7 @@ class BumpPtrAllocator {
StackBase = reinterpret_cast<uint8_t *>(
__mmap(0, MaxSize, PROT_READ | PROT_WRITE,
(Shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0));
- assert(StackBase != MAP_FAILED,
- "BumpPtrAllocator: failed to mmap stack!");
+ assert(!isErrValue(StackBase), "BumpPtrAllocator: failed to mmap stack!");
StackSize = 0;
}
@@ -759,7 +758,7 @@ ProfileWriterContext readDescriptions(const uint8_t *BinContents,
Size = __lseek(FD, 0, SEEK_END);
BinContents = reinterpret_cast<uint8_t *>(
__mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
- assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");
+ assert(!isErrValue(BinContents), "readDescriptions: Failed to mmap self!");
}
Result.MMapPtr = BinContents;
Result.MMapSize = Size;
@@ -1678,11 +1677,11 @@ extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {
void *Ret =
__mmap(CountersStart, CountersEnd - CountersStart, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);
- assert(Ret != MAP_FAILED, "__bolt_instr_setup: Failed to mmap counters!");
+ assert(!isErrValue(Ret), "__bolt_instr_setup: Failed to mmap counters!");
GlobalMetadataStorage = __mmap(0, 4096, PROT_READ | PROT_WRITE,
MapPrivateOrShared | MAP_ANONYMOUS, -1, 0);
- assert(GlobalMetadataStorage != MAP_FAILED,
+ assert(!isErrValue(GlobalMetadataStorage),
"__bolt_instr_setup: failed to mmap page for metadata!");
GlobalAlloc = new (GlobalMetadataStorage) BumpPtrAllocator;
>From a129edf07bec66c3534bb722b9980c66b82d3d64 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 9 Jun 2026 05:10:15 -0400
Subject: [PATCH 3/6] [BOLT] hugify: check __madvise return value correctly
The only error checked for the __madvise syscall wrapper was EPERM (-1).
---
bolt/runtime/hugify.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index 66c376305e689..ac28775e873b6 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -125,7 +125,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
}
// Mark the hot code page to be huge page.
- if (__madvise(From, Size, MADV_HUGEPAGE) == -1) {
+ if (__madvise(From, Size, MADV_HUGEPAGE) < 0) {
char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
@@ -160,9 +160,9 @@ extern "C" void __bolt_hugify_self_impl() {
return;
}
- if (__madvise(From, (To - From), MADV_HUGEPAGE) == -1) {
- char Msg[] = "[hugify] failed to allocate large page\n";
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) < 0) {
// TODO: allow user to control the failure behavior.
+ char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
}
>From d5dad9bbd5ce6553c519da83d83c1d5d1054e87f Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 9 Jun 2026 05:30:17 -0400
Subject: [PATCH 4/6] [BOLT] hugify: remap code as executable
The hugified process will crash with a segfault if the subsequent mprotect fails.
---
bolt/runtime/hugify.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index ac28775e873b6..1fff72c241590 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -115,9 +115,9 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
__prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- const void *Addr =
- __mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ const void *Addr = __mmap(reinterpret_cast<uint64_t>(From), Size,
+ PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (isErrValue(Addr)) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
>From 6565ca8c73f8e127801fe54983a1a4d443d24b93 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 5 May 2026 06:50:58 -0400
Subject: [PATCH 5/6] [BOLT] hugify: fix conflict with --no-huge-pages
There is a conflict between --hugify and --no-huge-pages options.
Both of these options change page alignment in different ways.
---
bolt/lib/Core/BinaryContext.cpp | 11 ++++++++++-
bolt/test/runtime/hugify.c | 20 ++++++++++++++++----
2 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 43ce56ef083b5..4902a8e7fa91b 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -158,7 +158,16 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
MIA(std::move(MIA)), MIB(std::move(MIB)), MRI(std::move(MRI)),
DisAsm(std::move(DisAsm)), Logger(Logger), InitialDynoStats(isAArch64()) {
RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
- PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
+ if (opts::Hugify) {
+ PageAlign = HugePageSize;
+ if (opts::NoHugePages) {
+ this->errs() << "BOLT-WARNING: a conflict between --hugify and "
+ "--no-huge-pages options, --no-huge-pages was skipped\n";
+ opts::NoHugePages = false;
+ }
+ } else {
+ PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
+ }
}
BinaryContext::~BinaryContext() {
diff --git a/bolt/test/runtime/hugify.c b/bolt/test/runtime/hugify.c
index a4a718a1160df..353dc9d7e39c0 100644
--- a/bolt/test/runtime/hugify.c
+++ b/bolt/test/runtime/hugify.c
@@ -14,15 +14,24 @@ RUN: %clang %cflags -no-pie %s -o %t.nopie.exe -Wl,-q
RUN: %clang %cflags -fpic %s -o %t.pie.exe -Wl,-q
RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie --hugify
+RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie.nhp --hugify \
+RUN: --no-huge-pages=true
RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie --hugify
+RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie.nhp --hugify -no-huge-pages=true
RUN: llvm-nm --numeric-sort --print-armap %t.nopie | \
RUN: FileCheck %s -check-prefix=CHECK-NM
-RUN: %t.nopie | FileCheck %s -check-prefix=CHECK-NOPIE
+RUN: %t.nopie | FileCheck %s -check-prefix=CHECK
+RUN: %t.nopie.nhp | FileCheck %s -check-prefix=CHECK
+RUN: llvm-readelf -lS %t.nopie | FileCheck %s -check-prefix=CHECK-ALIGNMENT
+RUN: llvm-readelf -lS %t.nopie.nhp | FileCheck %s -check-prefix=CHECK-ALIGNMENT
RUN: llvm-nm --numeric-sort --print-armap %t.pie | \
RUN: FileCheck %s -check-prefix=CHECK-NM
-RUN: %t.pie | FileCheck %s -check-prefix=CHECK-PIE
+RUN: %t.pie | FileCheck %s -check-prefix=CHECK
+RUN: %t.pie.nhp | FileCheck %s -check-prefix=CHECK
+RUN: llvm-readelf -lS %t.pie | FileCheck %s -check-prefix=CHECK-ALIGNMENT
+RUN: llvm-readelf -lS %t.pie.nhp | FileCheck %s -check-prefix=CHECK-ALIGNMENT
CHECK-NM: W __hot_start
CHECK-NM-NEXT: T _start
@@ -31,8 +40,11 @@ CHECK-NM: W __hot_end
CHECK-NM: t __bolt_hugify_start_program
CHECK-NM-NEXT: W __bolt_runtime_start
-CHECK-NOPIE: Hello world
+COM: .text section must have a hugepage alignment
+COM: at least one R-E segment must have a hugepage alignment
+CHECK-ALIGNMENT: {{.*}} .text PROGBITS {{[0-9a-f]+}}00000 {{.*}} 2097152
+CHECK-ALIGNMENT: LOAD 0x{{.*}} R E 0x200000
-CHECK-PIE: Hello world
+CHECK: Hello world
*/
>From 5b41ff935dab3398365ab1393a6bd14fc9ac8043 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 9 Jun 2026 06:59:48 -0400
Subject: [PATCH 6/6] [BOLT] hugify: refactor hasPagecacheTHPSupport function
The hasPagecacheTHPSupport function was used for two independent purposes:
- check whether THP is enabled via sysfs
- check whether THP for pagecache supported via the kernel version
It was refactored as follows:
1) separate function to check for THP through sysfs. Hugify is skipped
if THP is set to "[never]".
2) separate function to check pagecache support via kernel version.
Since the feature was merged into the 5.4 kernel version, the version
was updated.
See commit "mm,thp: add read-only THP support for (non-shmem) FS"
[https://github.com/torvalds/linux/commit/99cb0dbd47a15d35bf3faa78dc122bc5efe3fc0]
3) removed the warning related to the workaround for old kernels, as
nothing special is done for alignment in the hugify runtime library
4) fixed a file descriptor leakage
---
bolt/runtime/hugify.cpp | 62 ++++++++++++++++++++++-------------------
1 file changed, 34 insertions(+), 28 deletions(-)
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index 1fff72c241590..353da8dfb837e 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -59,40 +59,39 @@ static void getKernelVersion(uint32_t *Val) {
}
}
-/// Check whether the kernel supports THP via corresponding sysfs entry.
-/// thp works only starting from 5.10
-static bool hasPagecacheTHPSupport() {
+/// Check whether the THP enabled via corresponding sysfs entry.
+static bool isThpEnabled() {
char Buf[64];
+ bool ThpEnabled = false;
const int FD =
__open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY, 0);
if (FD < 0)
- return false;
+ return ThpEnabled;
memset(Buf, 0, sizeof(Buf));
const size_t Res = __read(FD, Buf, sizeof(Buf));
- if (Res < 0)
- return false;
+ if (Res > 0 && (strStr(Buf, "[always]") || strStr(Buf, "[madvise]")))
+ ThpEnabled = true;
- if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]")) {
- DEBUG(report("[hugify] THP support is not enabled.\n");)
- return false;
- }
+ __close(FD);
- struct KernelVersionTy {
- uint32_t major;
- uint32_t minor;
- uint32_t release;
- };
+ return ThpEnabled;
+}
- KernelVersionTy KernelVersion;
+/// Check whether the THP is supported for pagecache (read-only, non-shmem).
+/// The feature works only starting from 5.4
+static bool hasPagecacheTHPSupport() {
+ struct KernelVersionTy {
+ uint32_t major = 0;
+ uint32_t minor = 0;
+ uint32_t release = 0;
+ } KernelVersion;
getKernelVersion((uint32_t *)&KernelVersion);
- if (KernelVersion.major >= 6 ||
- (KernelVersion.major == 5 && KernelVersion.minor >= 10))
- return true;
- return false;
+ return KernelVersion.major >= 6 ||
+ (KernelVersion.major == 5 && KernelVersion.minor >= 4);
}
static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
@@ -153,17 +152,24 @@ extern "C" void __bolt_hugify_self_impl() {
DEBUG(reportNumber("[hugify] aligned huge page from: ", (uint64_t)From, 16);)
DEBUG(reportNumber("[hugify] aligned huge page to: ", (uint64_t)To, 16);)
- if (!hasPagecacheTHPSupport()) {
- DEBUG(report(
- "[hugify] workaround with memory alignment for kernel < 5.10\n");)
- hugifyForOldKernel(From, To);
+ // MADV_COLLAPSE (since Linux 6.1) ignores [never] state
+ if (!isThpEnabled()) {
+ DEBUG(report("[hugify] THP support is not enabled.\n");)
return;
}
- if (__madvise(From, (To - From), MADV_HUGEPAGE) < 0) {
- // TODO: allow user to control the failure behavior.
- char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
- reportError(Msg, sizeof(Msg));
+ if (hasPagecacheTHPSupport()) {
+ DEBUG(report("[hugify] THP for pagecache is supported.\n");)
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) < 0) {
+ // TODO: allow user to control the failure behavior.
+ char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
+ reportError(Msg, sizeof(Msg));
+ }
+
+ } else {
+ DEBUG(report("[hugify] THP for pagecache is not supported. The "
+ "copy-map-madvise approach is used\n");)
+ hugifyForOldKernel(From, To);
}
}
More information about the llvm-commits
mailing list