[llvm] Simple hugify fixes and refactoring (PR #202609)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 9 06:23:17 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-bolt
Author: DmitriiMartynov
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/202609.diff
5 Files Affected:
- (modified) bolt/lib/Core/BinaryContext.cpp (+10-1)
- (modified) bolt/runtime/common.h (+11-1)
- (modified) bolt/runtime/hugify.cpp (+46-41)
- (modified) bolt/runtime/instr.cpp (+4-5)
- (modified) bolt/test/runtime/hugify.c (+16-4)
``````````diff
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 43ce56ef083b5..4902a8e7fa91b 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -158,7 +158,16 @@ BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,
MIA(std::move(MIA)), MIB(std::move(MIB)), MRI(std::move(MRI)),
DisAsm(std::move(DisAsm)), Logger(Logger), InitialDynoStats(isAArch64()) {
RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;
- PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
+ if (opts::Hugify) {
+ PageAlign = HugePageSize;
+ if (opts::NoHugePages) {
+ this->errs() << "BOLT-WARNING: a conflict between --hugify and "
+ "--no-huge-pages options, --no-huge-pages was skipped\n";
+ opts::NoHugePages = false;
+ }
+ } else {
+ PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;
+ }
}
BinaryContext::~BinaryContext() {
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 8689bc8b72041..0014897ade6f9 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -62,7 +62,10 @@ typedef int int32_t;
#define MAP_ANONYMOUS 0x20
#endif
-#define MAP_FAILED ((void *)-1)
+#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
+
+/* set the state of the "THP disable" flags for the calling thread */
+#define PR_SET_THP_DISABLE 41
#define SEEK_SET 0 /* Seek from beginning of file. */
#define SEEK_CUR 1 /* Seek from current position. */
@@ -363,4 +366,11 @@ inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
return (Value + Align - 1) / Align * Align;
}
+// See include/linux/err.h file in the Linux kernel
+constexpr intptr_t MaxErrno = 4095;
+inline bool isErrValue(const void *Val) {
+ const intptr_t PtrVal = reinterpret_cast<intptr_t>(Val);
+ return PtrVal >= -MaxErrno && PtrVal <= -1;
+}
+
} // anonymous namespace
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index de896307f24fc..353da8dfb837e 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -59,50 +59,48 @@ static void getKernelVersion(uint32_t *Val) {
}
}
-/// Check whether the kernel supports THP via corresponding sysfs entry.
-/// thp works only starting from 5.10
-static bool hasPagecacheTHPSupport() {
+/// Check whether the THP enabled via corresponding sysfs entry.
+static bool isThpEnabled() {
char Buf[64];
+ bool ThpEnabled = false;
- int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
- 0 /* O_RDONLY */, 0);
+ const int FD =
+ __open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY, 0);
if (FD < 0)
- return false;
+ return ThpEnabled;
memset(Buf, 0, sizeof(Buf));
const size_t Res = __read(FD, Buf, sizeof(Buf));
- if (Res < 0)
- return false;
+ if (Res > 0 && (strStr(Buf, "[always]") || strStr(Buf, "[madvise]")))
+ ThpEnabled = true;
- if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]")) {
- DEBUG(report("[hugify] THP support is not enabled.\n");)
- return false;
- }
+ __close(FD);
- struct KernelVersionTy {
- uint32_t major;
- uint32_t minor;
- uint32_t release;
- };
+ return ThpEnabled;
+}
- KernelVersionTy KernelVersion;
+/// Check whether the THP is supported for pagecache (read-only, non-shmem).
+/// The feature works only starting from 5.4
+static bool hasPagecacheTHPSupport() {
+ struct KernelVersionTy {
+ uint32_t major = 0;
+ uint32_t minor = 0;
+ uint32_t release = 0;
+ } KernelVersion;
getKernelVersion((uint32_t *)&KernelVersion);
- if (KernelVersion.major >= 6 ||
- (KernelVersion.major == 5 && KernelVersion.minor >= 10))
- return true;
- return false;
+ return KernelVersion.major >= 6 ||
+ (KernelVersion.major == 5 && KernelVersion.minor >= 4);
}
static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
const size_t Size = To - From;
- uint8_t *Mem = reinterpret_cast<uint8_t *>(
- __mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
- 0x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
+ void *Mem = __mmap(0, Size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (Mem == ((void *)-1) /* MAP_FAILED */) {
+ if (isErrValue(Mem)) {
char Msg[] = "[hugify] could not allocate memory for text move\n";
reportError(Msg, sizeof(Msg));
}
@@ -114,19 +112,19 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
// Copy the hot code to a temporary location.
memcpy(Mem, From, Size);
- __prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
+ __prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- if (__mmap(reinterpret_cast<uint64_t>(From), Size,
- 0x3 /* PROT_READ | PROT_WRITE */,
- 0x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
- 0) == ((void *)-1) /*MAP_FAILED*/) {
+ const void *Addr = __mmap(reinterpret_cast<uint64_t>(From), Size,
+ PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ if (isErrValue(Addr)) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
reportError(Msg, sizeof(Msg));
}
// Mark the hot code page to be huge page.
- if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
+ if (__madvise(From, Size, MADV_HUGEPAGE) < 0) {
char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
@@ -135,7 +133,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
memcpy(From, Mem, Size);
// Change permission back to read-only, ignore failure
- __mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
+ __mprotect(From, Size, PROT_READ | PROT_EXEC);
__munmap(Mem, Size);
}
@@ -154,17 +152,24 @@ extern "C" void __bolt_hugify_self_impl() {
DEBUG(reportNumber("[hugify] aligned huge page from: ", (uint64_t)From, 16);)
DEBUG(reportNumber("[hugify] aligned huge page to: ", (uint64_t)To, 16);)
- if (!hasPagecacheTHPSupport()) {
- DEBUG(report(
- "[hugify] workaround with memory alignment for kernel < 5.10\n");)
- hugifyForOldKernel(From, To);
+ // MADV_COLLAPSE (since Linux 6.1) ignores [never] state
+ if (!isThpEnabled()) {
+ DEBUG(report("[hugify] THP support is not enabled.\n");)
return;
}
- if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
- char Msg[] = "[hugify] failed to allocate large page\n";
- // TODO: allow user to control the failure behavior.
- reportError(Msg, sizeof(Msg));
+ if (hasPagecacheTHPSupport()) {
+ DEBUG(report("[hugify] THP for pagecache is supported.\n");)
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) < 0) {
+ // TODO: allow user to control the failure behavior.
+ char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
+ reportError(Msg, sizeof(Msg));
+ }
+
+ } else {
+ DEBUG(report("[hugify] THP for pagecache is not supported. The "
+ "copy-map-madvise approach is used\n");)
+ hugifyForOldKernel(From, To);
}
}
diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp
index 3470e391224ba..e8b4542e4d2e3 100644
--- a/bolt/runtime/instr.cpp
+++ b/bolt/runtime/instr.cpp
@@ -141,8 +141,7 @@ class BumpPtrAllocator {
StackBase = reinterpret_cast<uint8_t *>(
__mmap(0, MaxSize, PROT_READ | PROT_WRITE,
(Shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0));
- assert(StackBase != MAP_FAILED,
- "BumpPtrAllocator: failed to mmap stack!");
+ assert(!isErrValue(StackBase), "BumpPtrAllocator: failed to mmap stack!");
StackSize = 0;
}
@@ -759,7 +758,7 @@ ProfileWriterContext readDescriptions(const uint8_t *BinContents,
Size = __lseek(FD, 0, SEEK_END);
BinContents = reinterpret_cast<uint8_t *>(
__mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
- assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");
+ assert(!isErrValue(BinContents), "readDescriptions: Failed to mmap self!");
}
Result.MMapPtr = BinContents;
Result.MMapSize = Size;
@@ -1678,11 +1677,11 @@ extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {
void *Ret =
__mmap(CountersStart, CountersEnd - CountersStart, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);
- assert(Ret != MAP_FAILED, "__bolt_instr_setup: Failed to mmap counters!");
+ assert(!isErrValue(Ret), "__bolt_instr_setup: Failed to mmap counters!");
GlobalMetadataStorage = __mmap(0, 4096, PROT_READ | PROT_WRITE,
MapPrivateOrShared | MAP_ANONYMOUS, -1, 0);
- assert(GlobalMetadataStorage != MAP_FAILED,
+ assert(!isErrValue(GlobalMetadataStorage),
"__bolt_instr_setup: failed to mmap page for metadata!");
GlobalAlloc = new (GlobalMetadataStorage) BumpPtrAllocator;
diff --git a/bolt/test/runtime/hugify.c b/bolt/test/runtime/hugify.c
index a4a718a1160df..353dc9d7e39c0 100644
--- a/bolt/test/runtime/hugify.c
+++ b/bolt/test/runtime/hugify.c
@@ -14,15 +14,24 @@ RUN: %clang %cflags -no-pie %s -o %t.nopie.exe -Wl,-q
RUN: %clang %cflags -fpic %s -o %t.pie.exe -Wl,-q
RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie --hugify
+RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie.nhp --hugify \
+RUN: --no-huge-pages=true
RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie --hugify
+RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie.nhp --hugify -no-huge-pages=true
RUN: llvm-nm --numeric-sort --print-armap %t.nopie | \
RUN: FileCheck %s -check-prefix=CHECK-NM
-RUN: %t.nopie | FileCheck %s -check-prefix=CHECK-NOPIE
+RUN: %t.nopie | FileCheck %s -check-prefix=CHECK
+RUN: %t.nopie.nhp | FileCheck %s -check-prefix=CHECK
+RUN: llvm-readelf -lS %t.nopie | FileCheck %s -check-prefix=CHECK-ALIGNMENT
+RUN: llvm-readelf -lS %t.nopie.nhp | FileCheck %s -check-prefix=CHECK-ALIGNMENT
RUN: llvm-nm --numeric-sort --print-armap %t.pie | \
RUN: FileCheck %s -check-prefix=CHECK-NM
-RUN: %t.pie | FileCheck %s -check-prefix=CHECK-PIE
+RUN: %t.pie | FileCheck %s -check-prefix=CHECK
+RUN: %t.pie.nhp | FileCheck %s -check-prefix=CHECK
+RUN: llvm-readelf -lS %t.pie | FileCheck %s -check-prefix=CHECK-ALIGNMENT
+RUN: llvm-readelf -lS %t.pie.nhp | FileCheck %s -check-prefix=CHECK-ALIGNMENT
CHECK-NM: W __hot_start
CHECK-NM-NEXT: T _start
@@ -31,8 +40,11 @@ CHECK-NM: W __hot_end
CHECK-NM: t __bolt_hugify_start_program
CHECK-NM-NEXT: W __bolt_runtime_start
-CHECK-NOPIE: Hello world
+COM: .text section must have a hugepage alignment
+COM: at least one R-E segment must have a hugepage alignment
+CHECK-ALIGNMENT: {{.*}} .text PROGBITS {{[0-9a-f]+}}00000 {{.*}} 2097152
+CHECK-ALIGNMENT: LOAD 0x{{.*}} R E 0x200000
-CHECK-PIE: Hello world
+CHECK: Hello world
*/
``````````
</details>
https://github.com/llvm/llvm-project/pull/202609
More information about the llvm-commits
mailing list