[llvm] [DRAFT][BOLT] runtime: syscall wrappers cleanup (PR #207190)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 2 06:51:42 PDT 2026
https://github.com/DmitriiMartynov created https://github.com/llvm/llvm-project/pull/207190
The BOLT runtime now uses [syscall wrapper implementations](https://github.com/llvm/llvm-project/blob/main/libc/src/__support/OSUtil/linux/syscall.h) from LLVM's libc. This
approach allows to drop arch-specific syscall implementations from the BOLT
runtime, reducing maintenance overhead.
The specific syscall implementations ([read](https://github.com/llvm/llvm-project/blob/main/libc/src/__support/OSUtil/linux/syscall_wrappers/read.h), [write](https://github.com/llvm/llvm-project/blob/main/libc/src/__support/OSUtil/linux/syscall_wrappers/write.h), etc.) from LLVM's libc are
not used in the BOLT runtime to avoid ErrorOr-related complexity. In LLVM libc,
syscall wrappers use ErrorOr to handle errors: they set errno and return -1 on
failure, or return a valid value otherwise. Since the BOLT runtime does not
support errno, it is simpler to use raw syscall wrappers and check for negative
return values to detect errors.
All syscall wrapper return values and argument types are now based on LLVM's
libc implementation:
- Return values are now signed for proper error checking.
- Argument types have been updated to match libc definitions.
Functionality of __exit, __open, and __readlink have been unified across
architectures. Except for __fork, which will be cleaned up later
>From e9f5028c0fd700d78fffc42c0420c108066cc868 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Tue, 5 May 2026 09:25:41 -0400
Subject: [PATCH 1/3] [BOLT] hugify: macro constants instead of hardcoded
values
All hardcoded values have been replaced with macro constants defined
in bolt/runtime/common.h to improve code readability.
---
bolt/runtime/common.h | 5 +++++
bolt/runtime/hugify.cpp | 25 +++++++++++--------------
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 96e269e1bb79b..cf300b266ae6c 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -64,6 +64,11 @@ typedef int int32_t;
#define MAP_FAILED ((void *)-1)
+#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
+
+/* set the state of the "THP disable" flags for the calling thread */
+#define PR_SET_THP_DISABLE 41
+
#define SEEK_SET 0 /* Seek from beginning of file. */
#define SEEK_CUR 1 /* Seek from current position. */
#define SEEK_END 2 /* Seek from end of file. */
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index de896307f24fc..34edf3cb1d03b 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -64,8 +64,8 @@ static void getKernelVersion(uint32_t *Val) {
static bool hasPagecacheTHPSupport() {
char Buf[64];
- int FD = __open("/sys/kernel/mm/transparent_hugepage/enabled",
- 0 /* O_RDONLY */, 0);
+ const int FD =
+ __open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY, 0);
if (FD < 0)
return false;
@@ -98,11 +98,10 @@ static bool hasPagecacheTHPSupport() {
static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
const size_t Size = To - From;
- uint8_t *Mem = reinterpret_cast<uint8_t *>(
- __mmap(0, Size, 0x3 /* PROT_READ | PROT_WRITE */,
- 0x22 /* MAP_PRIVATE | MAP_ANONYMOUS */, -1, 0));
+ void *Mem = __mmap(0, Size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (Mem == ((void *)-1) /* MAP_FAILED */) {
+ if (Mem == MAP_FAILED) {
char Msg[] = "[hugify] could not allocate memory for text move\n";
reportError(Msg, sizeof(Msg));
}
@@ -114,19 +113,17 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
// Copy the hot code to a temporary location.
memcpy(Mem, From, Size);
- __prctl(41 /* PR_SET_THP_DISABLE */, 0, 0, 0, 0);
+ __prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- if (__mmap(reinterpret_cast<uint64_t>(From), Size,
- 0x3 /* PROT_READ | PROT_WRITE */,
- 0x32 /* MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE */, -1,
- 0) == ((void *)-1) /*MAP_FAILED*/) {
+ if (__mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0) == MAP_FAILED) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
reportError(Msg, sizeof(Msg));
}
// Mark the hot code page to be huge page.
- if (__madvise(From, Size, 14 /* MADV_HUGEPAGE */) == -1) {
+ if (__madvise(From, Size, MADV_HUGEPAGE) == -1) {
char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
@@ -135,7 +132,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
memcpy(From, Mem, Size);
// Change permission back to read-only, ignore failure
- __mprotect(From, Size, 0x5 /* PROT_READ | PROT_EXEC */);
+ __mprotect(From, Size, PROT_READ | PROT_EXEC);
__munmap(Mem, Size);
}
@@ -161,7 +158,7 @@ extern "C" void __bolt_hugify_self_impl() {
return;
}
- if (__madvise(From, (To - From), 14 /* MADV_HUGEPAGE */) == -1) {
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) == -1) {
char Msg[] = "[hugify] failed to allocate large page\n";
// TODO: allow user to control the failure behavior.
reportError(Msg, sizeof(Msg));
>From 9cb2335e379ca5ba52fec1ee0197aad48bac9dea Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Fri, 26 Jun 2026 18:16:01 +0300
Subject: [PATCH 2/3] [BOLT] runtime: check syscall return values correctly for
errors
The return value of syscall wrappers in case of errors is not just -1, but a set
of negative values. Therefore, the correct way to check for errors is to test
whether the return value is negative.
The return value of the __mmap syscall wrapper is either a valid address or
an error (a negative value), not MAP_FAILED macro. The MAP_FAILED macro is
a libc return value of the mmap library function in case of an error where
the actual error is returned via errno variable. So, it is incorrect to compare
the __mmap syscall wrapper return value with MAP_FAILED, as only the EPERM (-1)
error is checked.
---
bolt/runtime/common.h | 16 ++++++++++++++--
bolt/runtime/hugify.cpp | 16 ++++++++--------
bolt/runtime/instr.cpp | 11 +++++------
3 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index cf300b266ae6c..3f37d82fec908 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -62,8 +62,6 @@ typedef int int32_t;
#define MAP_ANONYMOUS 0x20
#endif
-#define MAP_FAILED ((void *)-1)
-
#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
/* set the state of the "THP disable" flags for the calling thread */
@@ -368,4 +366,18 @@ inline uint64_t alignTo(uint64_t Value, uint64_t Align) {
return (Value + Align - 1) / Align * Align;
}
+constexpr intptr_t MaxErrno = 4095;
+// The function is used to detect errors from syscall wrappers that return
+// pointers instead of scalar values (for example, __mmap).
+// The return value of the __mmap wrapper is either a valid address or an error
+// (a negative value), not MAP_FAILED macro. The MAP_FAILED macro is a libc
+// return value of the mmap library function in case of an error where the
+// actual error is returned via errno variable. So, it is incorrect to compare
+// the __mmap syscall wrapper return value with MAP_FAILED, as only the EPERM
+// (-1) error is checked.
+inline bool isErrValue(const void *Val) {
+ const intptr_t PtrVal = reinterpret_cast<intptr_t>(Val);
+ return PtrVal >= -MaxErrno && PtrVal <= -1;
+}
+
} // anonymous namespace
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index 34edf3cb1d03b..af49ae4e713d0 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -70,7 +70,7 @@ static bool hasPagecacheTHPSupport() {
return false;
memset(Buf, 0, sizeof(Buf));
- const size_t Res = __read(FD, Buf, sizeof(Buf));
+ const long Res = __read(FD, Buf, sizeof(Buf));
if (Res < 0)
return false;
@@ -100,8 +100,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
void *Mem = __mmap(0, Size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-
- if (Mem == MAP_FAILED) {
+ if (isErrValue(Mem)) {
char Msg[] = "[hugify] could not allocate memory for text move\n";
reportError(Msg, sizeof(Msg));
}
@@ -115,15 +114,16 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
__prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- if (__mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
- MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0) == MAP_FAILED) {
+ const void *Addr = __mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
+ MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ if (isErrValue(Addr)) {
char Msg[] =
"[hugify] failed to mmap memory for large page move terminating\n";
reportError(Msg, sizeof(Msg));
}
// Mark the hot code page to be huge page.
- if (__madvise(From, Size, MADV_HUGEPAGE) == -1) {
+ if (__madvise(From, Size, MADV_HUGEPAGE) < 0) {
char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
reportError(Msg, sizeof(Msg));
}
@@ -158,8 +158,8 @@ extern "C" void __bolt_hugify_self_impl() {
return;
}
- if (__madvise(From, (To - From), MADV_HUGEPAGE) == -1) {
- char Msg[] = "[hugify] failed to allocate large page\n";
+ if (__madvise(From, (To - From), MADV_HUGEPAGE) < 0) {
+ char Msg[] = "[hugify] setting MADV_HUGEPAGE is failed\n";
// TODO: allow user to control the failure behavior.
reportError(Msg, sizeof(Msg));
}
diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp
index 1e5bddee5793e..ae327758e5719 100644
--- a/bolt/runtime/instr.cpp
+++ b/bolt/runtime/instr.cpp
@@ -141,8 +141,7 @@ class BumpPtrAllocator {
StackBase = reinterpret_cast<uint8_t *>(
__mmap(0, MaxSize, PROT_READ | PROT_WRITE,
(Shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0));
- assert(StackBase != MAP_FAILED,
- "BumpPtrAllocator: failed to mmap stack!");
+ assert(!isErrValue(StackBase), "BumpPtrAllocator: failed to mmap stack!");
StackSize = 0;
}
@@ -709,7 +708,7 @@ static char *getBinaryPath() {
"failed to open /proc/self/map_files");
while (long Nread = __getdents64(FDdir, (struct dirent64 *)Buf, BufSize)) {
- assert(static_cast<int64_t>(Nread) != -1, "failed to get folder entries");
+ assert(Nread >= 0, "failed to get folder entries");
struct dirent64 *d;
for (long Bpos = 0; Bpos < Nread; Bpos += d->d_reclen) {
@@ -759,7 +758,7 @@ ProfileWriterContext readDescriptions(const uint8_t *BinContents,
Size = __lseek(FD, 0, SEEK_END);
BinContents = reinterpret_cast<uint8_t *>(
__mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
- assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");
+ assert(!isErrValue(BinContents), "readDescriptions: Failed to mmap self!");
}
Result.MMapPtr = BinContents;
Result.MMapSize = Size;
@@ -1676,11 +1675,11 @@ extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {
void *Ret =
__mmap(CountersStart, CountersEnd - CountersStart, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);
- assert(Ret != MAP_FAILED, "__bolt_instr_setup: Failed to mmap counters!");
+ assert(!isErrValue(Ret), "__bolt_instr_setup: Failed to mmap counters!");
GlobalMetadataStorage = __mmap(0, 4096, PROT_READ | PROT_WRITE,
MapPrivateOrShared | MAP_ANONYMOUS, -1, 0);
- assert(GlobalMetadataStorage != MAP_FAILED,
+ assert(!isErrValue(GlobalMetadataStorage),
"__bolt_instr_setup: failed to mmap page for metadata!");
GlobalAlloc = new (GlobalMetadataStorage) BumpPtrAllocator;
>From e41ab85de3a656369876044e9682454670186267 Mon Sep 17 00:00:00 2001
From: Dmitrii Martynov <martynovdmitryvladimirovich at yandex.ru>
Date: Fri, 26 Jun 2026 18:16:01 +0300
Subject: [PATCH 3/3] [DRAFT][BOLT] runtime: syscall wrappers cleanup
The BOLT runtime now uses syscall wrapper implementations from LLVM's libc. This
approach allows to drop arch-specific syscall implementations from the BOLT
runtime, reducing maintenance overhead.
The specific syscall implementations (read, write, etc.) from LLVM's libc are
not used in the BOLT runtime to avoid ErrorOr-related complexity. In LLVM libc,
syscall wrappers use ErrorOr to handle errors: they set errno and return -1 on
failure, or return a valid value otherwise. Since the BOLT runtime does not
support errno, it is simpler to use raw syscall wrappers and check for negative
return values to detect errors.
All syscall wrapper return values and argument types are now based on LLVM's
libc implementation:
- Return values are now signed for proper error checking.
- Argument types have been updated to match libc definitions.
Functionality of __exit, __open, and __readlink have been unified across
architectures. Except for __fork, which will be cleaned up later.
---
bolt/runtime/CMakeLists.txt | 7 +
bolt/runtime/common.h | 126 +----------
bolt/runtime/hugify.cpp | 7 +-
bolt/runtime/instr.cpp | 54 +++--
bolt/runtime/runtime_types.h | 149 +++++++++++++
bolt/runtime/sys_aarch64.h | 371 ++++---------------------------
bolt/runtime/sys_riscv64.h | 375 ++++----------------------------
bolt/runtime/sys_x86_64.h | 353 +++++-------------------------
bolt/runtime/syscall_wrappers.h | 208 ++++++++++++++++++
9 files changed, 534 insertions(+), 1116 deletions(-)
create mode 100644 bolt/runtime/runtime_types.h
create mode 100644 bolt/runtime/syscall_wrappers.h
diff --git a/bolt/runtime/CMakeLists.txt b/bolt/runtime/CMakeLists.txt
index 63f178bd263c2..01a617a4c841e 100644
--- a/bolt/runtime/CMakeLists.txt
+++ b/bolt/runtime/CMakeLists.txt
@@ -64,6 +64,13 @@ target_include_directories(bolt_rt_instr PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
target_compile_options(bolt_rt_hugify PRIVATE ${BOLT_RT_FLAGS})
target_include_directories(bolt_rt_hugify PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+get_filename_component(LIBC_INCLUDE
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../libc
+ ABSOLUTE
+)
+target_include_directories(bolt_rt_instr PRIVATE ${LIBC_INCLUDE})
+target_include_directories(bolt_rt_hugify PRIVATE ${LIBC_INCLUDE})
+
install(TARGETS bolt_rt_instr DESTINATION "lib${LLVM_LIBDIR_SUFFIX}")
install(TARGETS bolt_rt_hugify DESTINATION "lib${LLVM_LIBDIR_SUFFIX}")
diff --git a/bolt/runtime/common.h b/bolt/runtime/common.h
index 3f37d82fec908..dbbdb15369b41 100644
--- a/bolt/runtime/common.h
+++ b/bolt/runtime/common.h
@@ -5,79 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-
-#if defined(__linux__)
-
-#include <cstddef>
-#include <cstdint>
-
-#include "config.h"
-
-#ifdef HAVE_ELF_H
-#include <elf.h>
-#endif
-
-#elif defined(__APPLE__)
-
-typedef __SIZE_TYPE__ size_t;
-#define __SSIZE_TYPE__ \
- __typeof__(_Generic((__SIZE_TYPE__)0, unsigned long long int \
- : (long long int)0, unsigned long int \
- : (long int)0, unsigned int \
- : (int)0, unsigned short \
- : (short)0, unsigned char \
- : (signed char)0))
-typedef __SSIZE_TYPE__ ssize_t;
-
-typedef unsigned long long uint64_t;
-typedef unsigned uint32_t;
-typedef unsigned char uint8_t;
-
-typedef long long int64_t;
-typedef int int32_t;
-
-#else
-#error "For Linux or MacOS only"
-#endif
-
-#define PROT_READ 0x1 /* Page can be read. */
-#define PROT_WRITE 0x2 /* Page can be written. */
-#define PROT_EXEC 0x4 /* Page can be executed. */
-#define PROT_NONE 0x0 /* Page can not be accessed. */
-#define PROT_GROWSDOWN \
- 0x01000000 /* Extend change to start of \
- growsdown vma (mprotect only). */
-#define PROT_GROWSUP \
- 0x02000000 /* Extend change to start of \
- growsup vma (mprotect only). */
-
-/* Sharing types (must choose one and only one of these). */
-#define MAP_SHARED 0x01 /* Share changes. */
-#define MAP_PRIVATE 0x02 /* Changes are private. */
-#define MAP_FIXED 0x10 /* Interpret addr exactly. */
-
-#if defined(__APPLE__)
-#define MAP_ANONYMOUS 0x1000
-#else
-#define MAP_ANONYMOUS 0x20
-#endif
-
-#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
-
-/* set the state of the "THP disable" flags for the calling thread */
-#define PR_SET_THP_DISABLE 41
-
-#define SEEK_SET 0 /* Seek from beginning of file. */
-#define SEEK_CUR 1 /* Seek from current position. */
-#define SEEK_END 2 /* Seek from end of file. */
-
-#define O_RDONLY 0
-#define O_WRONLY 1
-#define O_RDWR 2
-#define O_CREAT 64
-#define O_TRUNC 512
-#define O_APPEND 1024
-#define O_CLOEXEC 524288
+#include "runtime_types.h"
+#include "syscall_wrappers.h"
// Functions that are required by freestanding environment. Compiler may
// generate calls to these implicitly.
@@ -127,44 +56,11 @@ int memcmp(const void *s1, const void *s2, size_t n) {
// Anonymous namespace covering everything but our library entry point
namespace {
-struct dirent64 {
- uint64_t d_ino; /* Inode number */
- int64_t d_off; /* Offset to next linux_dirent */
- unsigned short d_reclen; /* Length of this linux_dirent */
- unsigned char d_type;
- char d_name[]; /* Filename (null-terminated) */
- /* length is actually (d_reclen - 2 -
- offsetof(struct linux_dirent, d_name)) */
-};
-
-/* Length of the entries in `struct utsname' is 65. */
-#define _UTSNAME_LENGTH 65
-
-struct UtsNameTy {
- char sysname[_UTSNAME_LENGTH]; /* Operating system name (e.g., "Linux") */
- char nodename[_UTSNAME_LENGTH]; /* Name within "some implementation-defined
- network" */
- char release[_UTSNAME_LENGTH]; /* Operating system release (e.g., "2.6.28") */
- char version[_UTSNAME_LENGTH]; /* Operating system version */
- char machine[_UTSNAME_LENGTH]; /* Hardware identifier */
- char domainname[_UTSNAME_LENGTH]; /* NIS or YP domain name */
-};
-
-struct timespec {
- uint64_t tv_sec; /* seconds */
- uint64_t tv_nsec; /* nanoseconds */
-};
-
-#if defined(__aarch64__) || defined(__arm64__)
-#include "sys_aarch64.h"
-#elif defined(__riscv)
-#include "sys_riscv64.h"
-#elif defined(__x86_64__)
-#include "sys_x86_64.h"
-#else
-#error "For AArch64/ARM64,X86_64 AND RISCV64 only."
-#endif
-
+// We use a stack-allocated buffer for string manipulation in many pieces of
+// this code, including the code that prints each line of the fdata file. This
+// buffer needs to accommodate large function names, but shouldn't be
+// arbitrarily large (dynamically allocated) for simplicity of our memory space
+// usage.
constexpr uint32_t BufSize = 32768U;
// Helper functions for writing strings to the .fdata file. We intentionally
@@ -309,11 +205,7 @@ void assert(bool Assertion, const char *Msg) {
#endif
}
-#define SIG_BLOCK 0
-#define SIG_UNBLOCK 1
-#define SIG_SETMASK 2
-
-static const uint64_t MaskAllSignals[] = {-1ULL};
+static const sigset_t MaskAllSignals[] = {-1ULL};
class Mutex {
volatile bool InUse{false};
@@ -326,7 +218,7 @@ class Mutex {
/// RAII wrapper for Mutex
class Lock {
Mutex &M;
- uint64_t SignalMask[1] = {};
+ sigset_t SignalMask[1] = {};
public:
Lock(Mutex &M) : M(M) {
diff --git a/bolt/runtime/hugify.cpp b/bolt/runtime/hugify.cpp
index af49ae4e713d0..234d26425fdc8 100644
--- a/bolt/runtime/hugify.cpp
+++ b/bolt/runtime/hugify.cpp
@@ -38,6 +38,9 @@ static void getKernelVersion(uint32_t *Val) {
// major, minor, release
struct UtsNameTy UtsName;
int Ret = __uname(&UtsName);
+ if (Ret < 0) {
+ // handle here
+ }
const char *Buf = UtsName.release;
const char *End = Buf + strLen(Buf);
const char Delims[2][2] = {".", "."};
@@ -70,7 +73,7 @@ static bool hasPagecacheTHPSupport() {
return false;
memset(Buf, 0, sizeof(Buf));
- const long Res = __read(FD, Buf, sizeof(Buf));
+ const ssize_t Res = __read(FD, Buf, sizeof(Buf));
if (Res < 0)
return false;
@@ -114,7 +117,7 @@ static void hugifyForOldKernel(uint8_t *From, uint8_t *To) {
__prctl(PR_SET_THP_DISABLE, 0, 0, 0, 0);
// Maps out the existing hot code.
- const void *Addr = __mmap(reinterpret_cast<uint64_t>(From), Size, PROT_READ | PROT_WRITE,
+ const void *Addr = __mmap(From, Size, PROT_READ | PROT_WRITE,
MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
if (isErrValue(Addr)) {
char Msg[] =
diff --git a/bolt/runtime/instr.cpp b/bolt/runtime/instr.cpp
index ae327758e5719..390eb32c65cbb 100644
--- a/bolt/runtime/instr.cpp
+++ b/bolt/runtime/instr.cpp
@@ -702,12 +702,10 @@ static char *getBinaryPath() {
return TargetPath;
unsigned long CurAddr = (unsigned long)__get_pc();
- uint64_t FDdir = __open(DirPath, O_RDONLY,
- /*mode=*/0666);
- assert(static_cast<int64_t>(FDdir) >= 0,
- "failed to open /proc/self/map_files");
+ int FDdir = __open(DirPath, O_RDONLY, /*mode=*/0666);
+ assert(FDdir >= 0, "failed to open /proc/self/map_files");
- while (long Nread = __getdents64(FDdir, (struct dirent64 *)Buf, BufSize)) {
+ while (ssize_t Nread = __getdents64(FDdir, (struct dirent64 *)Buf, BufSize)) {
assert(Nread >= 0, "failed to get folder entries");
struct dirent64 *d;
@@ -723,9 +721,8 @@ static char *getBinaryPath() {
char *C = strCopy(FindBuf, DirPath, NameMax);
C = strCopy(C, d->d_name, NameMax - (C - FindBuf));
*C = '\0';
- uint64_t Ret = __readlink(FindBuf, TargetPath, sizeof(TargetPath));
- assert(static_cast<int64_t>(Ret) >= 0 && Ret < sizeof(TargetPath),
- "readlink error");
+ ssize_t Ret = __readlink(FindBuf, TargetPath, sizeof(TargetPath));
+ assert(Ret >= 0 && Ret < sizeof(TargetPath), "readlink error");
TargetPath[Ret] = '\0';
__close(FDdir);
return TargetPath;
@@ -748,14 +745,15 @@ ProfileWriterContext readDescriptions(const uint8_t *BinContents,
const char *BinPath = getBinaryPath();
assert(BinPath && BinPath[0] != '\0', "failed to find binary path");
- uint64_t FD = __open(BinPath, O_RDONLY,
- /*mode=*/0666);
- assert(static_cast<int64_t>(FD) >= 0, "failed to open binary path");
+ int FD = __open(BinPath, O_RDONLY, /*mode=*/0666);
+ assert(FD >= 0, "failed to open binary path");
Result.FileDesc = FD;
// mmap our binary to memory
- Size = __lseek(FD, 0, SEEK_END);
+ off_t Ret = __lseek(FD, 0, SEEK_END);
+ assert(Ret >= 0, "Failed to lseek!");
+ Size = static_cast<uint64_t>(Ret);
BinContents = reinterpret_cast<uint8_t *>(
__mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));
assert(!isErrValue(BinContents), "readDescriptions: Failed to mmap self!");
@@ -1492,7 +1490,7 @@ void visitCallFlowEntry(CallFlowHashTable::MapEntry &Entry, int FD,
int openProfile() {
// Build the profile name string by appending our PID
char Buf[BufSize];
- uint64_t PID = __getpid();
+ pid_t PID = __getpid();
char *Ptr = strCopy(Buf, __bolt_instr_filename, BufSize);
if (__bolt_instr_use_pid) {
Ptr = strCopy(Ptr, ".", BufSize - (Ptr - Buf + 1));
@@ -1500,13 +1498,11 @@ int openProfile() {
Ptr = strCopy(Ptr, ".fdata", BufSize - (Ptr - Buf + 1));
}
*Ptr++ = '\0';
- uint64_t FD = __open(Buf, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC,
- /*mode=*/0600);
- if (static_cast<int64_t>(FD) < 0) {
+ int FD = __open(Buf, O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, /*mode=*/0600);
+ if (FD < 0) {
report("Error while trying to open profile file for writing: ");
report(Buf);
- reportNumber("\nFailed with error number: 0x",
- 0 - static_cast<int64_t>(FD), 16);
+ reportNumber("\nFailed with error number: 0x", FD, 16);
__exit(1);
}
return FD;
@@ -1569,10 +1565,10 @@ __bolt_instr_data_dump(int FD, const char *LibPath = nullptr,
if (!GlobalWriteProfileMutex->acquire())
return;
- int ret = __lseek(FD, 0, SEEK_SET);
- assert(ret == 0, "Failed to lseek!");
- ret = __ftruncate(FD, 0);
- assert(ret == 0, "Failed to ftruncate!");
+ off_t Ret = __lseek(FD, 0, SEEK_SET);
+ assert(Ret >= 0, "Failed to lseek!");
+ int RetTr = __ftruncate(FD, 0);
+ assert(RetTr >= 0, "Failed to ftruncate!");
BumpPtrAllocator HashAlloc;
HashAlloc.setMaxSize(__bolt_instr_max_size);
ProfileWriterContext Ctx = readDescriptions(LibContents, LibSize);
@@ -1610,10 +1606,10 @@ void watchProcess() {
timespec ts, rem;
uint64_t Elapsed = 0ull;
int FD = openProfile();
- uint64_t ppid;
+ int ppid;
if (__bolt_instr_wait_forks) {
// Store parent pgid
- ppid = -__getpgid(0);
+ ppid = __getpgid(0);
// And leave parent process group
__setpgid(0, 0);
} else {
@@ -1672,9 +1668,9 @@ extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {
const bool Shared = !__bolt_instr_use_pid | !!__bolt_instr_sleep_time;
const uint64_t MapPrivateOrShared = Shared ? MAP_SHARED : MAP_PRIVATE;
- void *Ret =
- __mmap(CountersStart, CountersEnd - CountersStart, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);
+ void *Ret = __mmap(reinterpret_cast<void *>(CountersStart),
+ CountersEnd - CountersStart, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);
assert(!isErrValue(Ret), "__bolt_instr_setup: Failed to mmap counters!");
GlobalMetadataStorage = __mmap(0, 4096, PROT_READ | PROT_WRITE,
@@ -1703,8 +1699,10 @@ extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {
if (__bolt_instr_wait_forks)
__setpgid(0, 0);
- if (long PID = __fork())
+ if (int PID = __fork()) {
+ assert(PID > 0, "__bolt_instr_setup: fork failed");
return;
+ }
watchProcess();
}
}
diff --git a/bolt/runtime/runtime_types.h b/bolt/runtime/runtime_types.h
new file mode 100644
index 0000000000000..c78887ba2e9cc
--- /dev/null
+++ b/bolt/runtime/runtime_types.h
@@ -0,0 +1,149 @@
+//===- bolt/runtime/runtime_types.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_BOLT_RUNTIME_TYPES
+#define LLVM_TOOLS_LLVM_BOLT_RUNTIME_TYPES
+
+// FIXME: cleanup here
+
+#if defined(__linux__)
+
+#include <cstddef>
+#include <cstdint>
+
+#include "config.h"
+
+#ifdef HAVE_ELF_H
+#include <elf.h>
+#endif
+
+#elif defined(__APPLE__)
+
+typedef __SIZE_TYPE__ size_t;
+#define __SSIZE_TYPE__ \
+ __typeof__(_Generic((__SIZE_TYPE__)0, \
+ unsigned long long int: (long long int)0, \
+ unsigned long int: (long int)0, \
+ unsigned int: (int)0, \
+ unsigned short: (short)0, \
+ unsigned char: (signed char)0))
+typedef __SSIZE_TYPE__ ssize_t;
+
+typedef unsigned long long uint64_t;
+typedef unsigned uint32_t;
+typedef unsigned char uint8_t;
+
+typedef long long int64_t;
+typedef int int32_t;
+
+#else
+#error "For Linux or MacOS only"
+#endif
+
+#ifndef __LP64__
+#error "Only LP64 data model is supported"
+#endif
+
+#define AT_FDCWD -100
+
+#define PROT_READ 0x1 /* Page can be read. */
+#define PROT_WRITE 0x2 /* Page can be written. */
+#define PROT_EXEC 0x4 /* Page can be executed. */
+#define PROT_NONE 0x0 /* Page can not be accessed. */
+#define PROT_GROWSDOWN \
+ 0x01000000 /* Extend change to start of \
+ growsdown vma (mprotect only). */
+#define PROT_GROWSUP \
+ 0x02000000 /* Extend change to start of \
+ growsup vma (mprotect only). */
+
+/* Sharing types (must choose one and only one of these). */
+#define MAP_SHARED 0x01 /* Share changes. */
+#define MAP_PRIVATE 0x02 /* Changes are private. */
+#define MAP_FIXED 0x10 /* Interpret addr exactly. */
+
+#if defined(__APPLE__)
+#define MAP_ANONYMOUS 0x1000
+#else
+#define MAP_ANONYMOUS 0x20
+#endif
+
+#define MADV_HUGEPAGE 14 /* Worth backing with hugepages */
+
+/* set the state of the "THP disable" flags for the calling thread */
+#define PR_SET_THP_DISABLE 41
+
+#define SEEK_SET 0 /* Seek from beginning of file. */
+#define SEEK_CUR 1 /* Seek from current position. */
+#define SEEK_END 2 /* Seek from end of file. */
+
+#define O_RDONLY 0
+#define O_WRONLY 1
+#define O_RDWR 2
+#define O_CREAT 64
+#define O_TRUNC 512
+#define O_APPEND 1024
+#define O_CLOEXEC 524288
+
+#define SIG_BLOCK 0
+#define SIG_UNBLOCK 1
+#define SIG_SETMASK 2
+
+#define CLONE_CHILD_CLEARTID 0x00200000 /* clear the TID in the child */
+#define CLONE_CHILD_SETTID 0x01000000 /* set the TID in the child */
+
+#define SIGCHLD 17
+
+/* Length of the entries in `struct utsname' is 65. */
+#define _UTSNAME_LENGTH 65
+
+#define _NSIG 64
+#define _NSIG_BPW 64
+#define _NSIG_WORDS (_NSIG / _NSIG_BPW)
+
+// x86_64, aarch64, riscv64 / Linux/Unix - LP64
+typedef long int ssize_t;
+typedef long int off_t;
+typedef int pid_t;
+typedef unsigned mode_t;
+
+// Anonymous namespace covering everything but our library entry point
+namespace {
+
+struct dirent64 {
+ uint64_t d_ino; /* Inode number */
+ int64_t d_off; /* Offset to next linux_dirent */
+ unsigned short d_reclen; /* Length of this linux_dirent */
+ unsigned char d_type;
+ char d_name[]; /* Filename (null-terminated) */
+ /* length is actually (d_reclen - 2 -
+ offsetof(struct linux_dirent, d_name)) */
+};
+
+struct UtsNameTy {
+ char sysname[_UTSNAME_LENGTH]; /* Operating system name (e.g., "Linux") */
+ char nodename[_UTSNAME_LENGTH]; /* Name within "some implementation-defined
+ network" */
+ char release[_UTSNAME_LENGTH]; /* Operating system release (e.g., "2.6.28") */
+ char version[_UTSNAME_LENGTH]; /* Operating system version */
+ char machine[_UTSNAME_LENGTH]; /* Hardware identifier */
+ char domainname[_UTSNAME_LENGTH]; /* NIS or YP domain name */
+};
+
+struct timespec {
+ uint64_t tv_sec; /* seconds */
+ uint64_t tv_nsec; /* nanoseconds */
+};
+
+typedef struct {
+ unsigned long sig[_NSIG_WORDS];
+} sigset_t;
+
+} // anonymous namespace
+
+#endif /* LLVM_TOOLS_LLVM_BOLT_RUNTIME_TYPES */
diff --git a/bolt/runtime/sys_aarch64.h b/bolt/runtime/sys_aarch64.h
index 9cb8e022f58df..8781b8f970d77 100644
--- a/bolt/runtime/sys_aarch64.h
+++ b/bolt/runtime/sys_aarch64.h
@@ -1,6 +1,16 @@
+//===- bolt/runtime/sys_aarch64.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
#ifndef LLVM_TOOLS_LLVM_BOLT_SYS_AARCH64
#define LLVM_TOOLS_LLVM_BOLT_SYS_AARCH64
+#include "runtime_types.h"
+
// Save all registers while keeping 16B stack alignment
#define SAVE_ALL \
"stp x0, x1, [sp, #-16]!\n" \
@@ -40,6 +50,36 @@
"ldp x2, x3, [sp], #16\n" \
"ldp x0, x1, [sp], #16\n"
+// https://github.com/torvalds/linux/blob/v7.1/arch/arm64/tools/syscall_64.tbl
+// https://github.com/torvalds/linux/blob/v7.1/scripts/syscall.tbl
+// scripts/syscalltbl.sh --abis common,64 arch/arm64/tools/syscall_64.tbl
+// arm64_syscalls.h
+#define __NR_ftruncate 46
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readlinkat 78
+#define __NR_fsync 82
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_nanosleep 101
+#define __NR_kill 129
+#define __NR_rt_sigprocmask 135
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_uname 160
+#define __NR_prctl 167
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_munmap 215
+#define __NR_clone 220
+#define __NR_mmap 222
+#define __NR_mprotect 226
+#define __NR_madvise 233
+
// Anonymous namespace covering everything but our library entry point
namespace {
@@ -62,335 +102,6 @@ uint64_t getTextBaseAddress() {
return DynAddr - StaticAddr;
}
-uint64_t __read(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
- register uint64_t x0 __asm__("x0") = fd;
- register const void *x1 __asm__("x1") = buf;
- register uint64_t x2 __asm__("x2") = count;
- register uint32_t w8 __asm__("w8") = 63;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
- register uint64_t x0 __asm__("x0") = fd;
- register const void *x1 __asm__("x1") = buf;
- register uint64_t x2 __asm__("x2") = count;
- register uint32_t w8 __asm__("w8") = 64;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags,
- uint64_t fd, uint64_t offset) {
- void *ret;
- register uint64_t x0 __asm__("x0") = addr;
- register uint64_t x1 __asm__("x1") = size;
- register uint64_t x2 __asm__("x2") = prot;
- register uint64_t x3 __asm__("x3") = flags;
- register uint64_t x4 __asm__("x4") = fd;
- register uint64_t x5 __asm__("x5") = offset;
- register uint32_t w8 __asm__("w8") = 222;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(x4), "r"(x5), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __munmap(void *addr, uint64_t size) {
- uint64_t ret;
- register void *x0 __asm__("x0") = addr;
- register uint64_t x1 __asm__("x1") = size;
- register uint32_t w8 __asm__("w8") = 215;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __exit(uint64_t code) {
- uint64_t ret;
- register uint64_t x0 __asm__("x0") = code;
- register uint32_t w8 __asm__("w8") = 94;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0)
- : "r"(w8)
- : "cc", "memory", "x1");
- return ret;
-}
-
-uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) {
- uint64_t ret;
- register int x0 __asm__("x0") = -100;
- register const char *x1 __asm__("x1") = pathname;
- register uint64_t x2 __asm__("x2") = flags;
- register uint64_t x3 __asm__("x3") = mode;
- register uint32_t w8 __asm__("w8") = 56;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-long __getdents64(unsigned int fd, dirent64 *dirp, size_t count) {
- long ret;
- register unsigned int x0 __asm__("x0") = fd;
- register dirent64 *x1 __asm__("x1") = dirp;
- register size_t x2 __asm__("x2") = count;
- register uint32_t w8 __asm__("w8") = 61;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) {
- uint64_t ret;
- register int x0 __asm__("x0") = -100;
- register const char *x1 __asm__("x1") = pathname;
- register char *x2 __asm__("x2") = buf;
- register size_t x3 __asm__("x3") = bufsize;
- register uint32_t w8 __asm__("w8") = 78; // readlinkat
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
- uint64_t ret;
- register uint64_t x0 __asm__("x0") = fd;
- register uint64_t x1 __asm__("x1") = pos;
- register uint64_t x2 __asm__("x2") = whence;
- register uint32_t w8 __asm__("w8") = 62;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __ftruncate(uint64_t fd, uint64_t length) {
- int ret;
- register uint64_t x0 __asm__("x0") = fd;
- register uint64_t x1 __asm__("x1") = length;
- register uint32_t w8 __asm__("w8") = 46;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __close(uint64_t fd) {
- int ret;
- register uint64_t x0 __asm__("x0") = fd;
- register uint32_t w8 __asm__("w8") = 57;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0)
- : "r"(w8)
- : "cc", "memory", "x1");
- return ret;
-}
-
-int __madvise(void *addr, size_t length, int advice) {
- int ret;
- register void *x0 __asm__("x0") = addr;
- register size_t x1 __asm__("x1") = length;
- register int x2 __asm__("x2") = advice;
- register uint32_t w8 __asm__("w8") = 233;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __uname(struct UtsNameTy *buf) {
- int ret;
- register UtsNameTy *x0 __asm__("x0") = buf;
- register uint32_t w8 __asm__("w8") = 160;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0)
- : "r"(w8)
- : "cc", "memory", "x1");
- return ret;
-}
-
-uint64_t __nanosleep(const timespec *req, timespec *rem) {
- uint64_t ret;
- register const timespec *x0 __asm__("x0") = req;
- register timespec *x1 __asm__("x1") = rem;
- register uint32_t w8 __asm__("w8") = 101;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int64_t __fork() {
- uint64_t ret;
- // clone instead of fork with flags
- // "CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD"
- register uint64_t x0 __asm__("x0") = 0x1200011;
- register uint64_t x1 __asm__("x1") = 0;
- register uint64_t x2 __asm__("x2") = 0;
- register uint64_t x3 __asm__("x3") = 0;
- register uint64_t x4 __asm__("x4") = 0;
- register uint32_t w8 __asm__("w8") = 220;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(x4), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __mprotect(void *addr, size_t len, int prot) {
- int ret;
- register void *x0 __asm__("x0") = addr;
- register size_t x1 __asm__("x1") = len;
- register int x2 __asm__("x2") = prot;
- register uint32_t w8 __asm__("w8") = 226;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __getpid() {
- uint64_t ret;
- register uint32_t w8 __asm__("w8") = 172;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret)
- : "r"(w8)
- : "cc", "memory", "x0", "x1");
- return ret;
-}
-
-uint64_t __getppid() {
- uint64_t ret;
- register uint32_t w8 __asm__("w8") = 173;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret)
- : "r"(w8)
- : "cc", "memory", "x0", "x1");
- return ret;
-}
-
-int __setpgid(uint64_t pid, uint64_t pgid) {
- int ret;
- register uint64_t x0 __asm__("x0") = pid;
- register uint64_t x1 __asm__("x1") = pgid;
- register uint32_t w8 __asm__("w8") = 154;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-uint64_t __getpgid(uint64_t pid) {
- uint64_t ret;
- register uint64_t x0 __asm__("x0") = pid;
- register uint32_t w8 __asm__("w8") = 155;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0)
- : "r"(w8)
- : "cc", "memory", "x1");
- return ret;
-}
-
-int __kill(uint64_t pid, int sig) {
- int ret;
- register uint64_t x0 __asm__("x0") = pid;
- register int x1 __asm__("x1") = sig;
- register uint32_t w8 __asm__("w8") = 129;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __fsync(int fd) {
- int ret;
- register int x0 __asm__("x0") = fd;
- register uint32_t w8 __asm__("w8") = 82;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0)
- : "r"(w8)
- : "cc", "memory", "x1");
- return ret;
-}
-
-uint64_t __sigprocmask(int how, const void *set, void *oldset) {
- uint64_t ret;
- register int x0 __asm__("x0") = how;
- register const void *x1 __asm__("x1") = set;
- register void *x2 __asm__("x2") = oldset;
- register long x3 asm("x3") = 8;
- register uint32_t w8 __asm__("w8") = 135;
- __asm__ __volatile__("svc #0\n"
- "mov %0, x0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
-int __prctl(int option, unsigned long arg2, unsigned long arg3,
- unsigned long arg4, unsigned long arg5) {
- int ret;
- register int x0 __asm__("x0") = option;
- register unsigned long x1 __asm__("x1") = arg2;
- register unsigned long x2 __asm__("x2") = arg3;
- register unsigned long x3 __asm__("x3") = arg4;
- register unsigned long x4 __asm__("x4") = arg5;
- register uint32_t w8 __asm__("w8") = 167;
- __asm__ __volatile__("svc #0\n"
- "mov %w0, w0"
- : "=r"(ret), "+r"(x0), "+r"(x1)
- : "r"(x2), "r"(x3), "r"(x4), "r"(w8)
- : "cc", "memory");
- return ret;
-}
-
} // anonymous namespace
-#endif
+#endif /* LLVM_TOOLS_LLVM_BOLT_SYS_AARCH64 */
diff --git a/bolt/runtime/sys_riscv64.h b/bolt/runtime/sys_riscv64.h
index b5ed64941e937..67ad22f9ccbf7 100644
--- a/bolt/runtime/sys_riscv64.h
+++ b/bolt/runtime/sys_riscv64.h
@@ -1,6 +1,16 @@
+//===- bolt/runtime/sys_riscv64.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
#ifndef LLVM_TOOLS_LLVM_BOLT_SYS_RISCV
#define LLVM_TOOLS_LLVM_BOLT_SYS_RISCV
+#include "runtime_types.h"
+
// Save all registers while keeping 16B stack alignment
#define SAVE_ALL \
"addi sp, sp, -256\n" \
@@ -70,6 +80,34 @@
"ld x31, 240(sp)\n" \
"addi sp, sp, 256\n"
+// https://github.com/torvalds/linux/blob/v7.1/scripts/syscall.tbl
+// scripts/syscalltbl.sh --abis common,64 scripts/syscall.tbl riscv64_syscalls.h
+#define __NR_ftruncate 46
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readlinkat 78
+#define __NR_fsync 82
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_nanosleep 101
+#define __NR_kill 129
+#define __NR_rt_sigprocmask 135
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_uname 160
+#define __NR_prctl 167
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_munmap 215
+#define __NR_clone 220
+#define __NR_mmap 222
+#define __NR_mprotect 226
+#define __NR_madvise 233
+
// Anonymous namespace covering everything but our library entry point
namespace {
@@ -88,341 +126,6 @@ uint64_t getTextBaseAddress() {
return DynAddr - StaticAddr;
}
-uint64_t __read(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
- register uint64_t a0 __asm__("a0") = fd;
- register const void *a1 __asm__("a1") = buf;
- register uint64_t a2 __asm__("a2") = count;
- register uint64_t a7 __asm__("a7") =
- 63; // Assuming 63 is the syscall number for read
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a0), "r"(a1), "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
- register uint64_t a0 __asm__("a0") = fd;
- register const void *a1 __asm__("a1") = buf;
- register uint64_t a2 __asm__("a2") = count;
- register uint32_t a7 __asm__("a7") =
- 64; // Assuming 64 is the syscall number for write
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a0), "r"(a1), "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags,
- uint64_t fd, uint64_t offset) {
- void *ret;
- register uint64_t a0 __asm__("a0") = addr;
- register uint64_t a1 __asm__("a1") = size;
- register uint64_t a2 __asm__("a2") = prot;
- register uint64_t a3 __asm__("a3") = flags;
- register uint64_t a4 __asm__("a4") = fd;
- register uint64_t a5 __asm__("a5") = offset;
- register uint32_t a7 __asm__("a7") =
- 222; // Assuming 222 is the syscall number for mmap
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5),
- "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __munmap(void *addr, uint64_t size) {
- uint64_t ret;
- register void *a0 __asm__("a0") = addr;
- register uint64_t a1 __asm__("a1") = size;
- register uint32_t a7 __asm__("a7") = 215;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __exit(uint64_t code) {
- uint64_t ret;
- register uint64_t a0 __asm__("a0") = code;
- register uint32_t a7 __asm__("a7") = 94;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) {
- uint64_t ret;
- register int a0 __asm__("a0") =
- -100; // Assuming -100 is an invalid file descriptor
- register const char *a1 __asm__("a1") = pathname;
- register uint64_t a2 __asm__("a2") = flags;
- register uint64_t a3 __asm__("a3") = mode;
- register uint64_t a7 __asm__("a7") =
- 56; // Assuming 56 is the syscall number for open
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(a7)
- : "memory");
- return ret;
-}
-
-long __getdents64(unsigned int fd, dirent64 *dirp, size_t count) {
- long ret;
- register unsigned int a0 __asm__("a0") = fd;
- register dirent64 *a1 __asm__("a1") = dirp;
- register size_t a2 __asm__("a2") = count;
- register uint32_t a7 __asm__("a7") = 61;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) {
- uint64_t ret;
- register int a0 __asm__("a0") = -100;
- register const char *a1 __asm__("a1") = pathname;
- register char *a2 __asm__("a2") = buf;
- register size_t a3 __asm__("a3") = bufsize;
- register uint32_t a7 __asm__("a7") = 78; // readlinkat
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a3), "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
- uint64_t ret;
- register uint64_t a0 __asm__("a0") = fd;
- register uint64_t a1 __asm__("a1") = pos;
- register uint64_t a2 __asm__("a2") = whence;
- register uint32_t a7 __asm__("a7") = 62;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-int __ftruncate(uint64_t fd, uint64_t length) {
- int ret;
- register uint64_t a0 __asm__("a0") = fd;
- register uint64_t a1 __asm__("a1") = length;
- register uint32_t a7 __asm__("a7") = 46;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-int __close(uint64_t fd) {
- int ret;
- register uint64_t a0 __asm__("a0") = fd;
- register uint32_t a7 __asm__("a7") =
- 57; // Assuming 57 is the syscall number for close
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a0), "r"(a7)
- : "memory");
- return ret;
-}
-
-int __madvise(void *addr, size_t length, int advice) {
- int ret;
- register void *a0 __asm__("a0") = addr;
- register size_t a1 __asm__("a1") = length;
- register int a2 __asm__("a2") = advice;
- register uint32_t a7 __asm__("a7") = 233;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-int __uname(struct UtsNameTy *buf) {
- int ret;
- register UtsNameTy *a0 __asm__("a0") = buf;
- register uint32_t a7 __asm__("a7") = 160;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __nanosleep(const timespec *req, timespec *rem) {
- uint64_t ret;
- register const timespec *a0 __asm__("a0") = req;
- register timespec *a1 __asm__("a1") = rem;
- register uint32_t a7 __asm__("a7") = 101;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-int64_t __fork() {
- uint64_t ret;
- // clone instead of fork with flags
- // "CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD"
- register uint64_t a0 __asm__("a0") = 0x1200011;
- register uint64_t a1 __asm__("a1") = 0;
- register uint64_t a2 __asm__("a2") = 0;
- register uint64_t a3 __asm__("a3") = 0;
- register uint64_t a4 __asm__("a4") = 0;
- register uint32_t a7 __asm__("a7") = 220;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a3), "r"(a4), "r"(a7)
- : "memory");
- return ret;
-}
-
-int __mprotect(void *addr, size_t len, int prot) {
- int ret;
- register void *a0 __asm__("a0") = addr;
- register size_t a1 __asm__("a1") = len;
- register int a2 __asm__("a2") = prot;
- register uint32_t a7 __asm__("a7") = 226;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __getpid() {
- uint64_t ret;
- register uint32_t a7 __asm__("a7") = 172;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __getppid() {
- uint64_t ret;
- register uint32_t a7 __asm__("a7") = 173;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-int __setpgid(uint64_t pid, uint64_t pgid) {
- int ret;
- register uint64_t a0 __asm__("a0") = pid;
- register uint64_t a1 __asm__("a1") = pgid;
- register uint32_t a7 __asm__("a7") = 154;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __getpgid(uint64_t pid) {
- uint64_t ret;
- register uint64_t a0 __asm__("a0") = pid;
- register uint32_t a7 __asm__("a7") = 155;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-int __kill(uint64_t pid, int sig) {
- int ret;
- register uint64_t a0 __asm__("a0") = pid;
- register int a1 __asm__("a1") = sig;
- register uint32_t a7 __asm__("a7") = 129;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-int __fsync(int fd) {
- int ret;
- register int a0 __asm__("a0") = fd;
- register uint32_t a7 __asm__("a7") = 82;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0)
- : "r"(a7)
- : "memory");
- return ret;
-}
-
-uint64_t __sigprocmask(int how, const void *set, void *oldset) {
- uint64_t ret;
- register int a0 __asm__("a0") = how;
- register const void *a1 __asm__("a1") = set;
- register void *a2 __asm__("a2") = oldset;
- register long a3 asm("a3") = 8;
- register uint32_t a7 __asm__("a7") = 135;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a3), "r"(a7)
- : "memory");
- return ret;
-}
-
-int __prctl(int option, unsigned long arg2, unsigned long arg3,
- unsigned long arg4, unsigned long arg5) {
- int ret;
- register int a0 __asm__("a0") = option;
- register unsigned long a1 __asm__("a1") = arg2;
- register unsigned long a2 __asm__("a2") = arg3;
- register unsigned long a3 __asm__("a3") = arg4;
- register unsigned long a4 __asm__("a4") = arg5;
- register uint32_t a7 __asm__("a7") = 167;
- __asm__ __volatile__("ecall\n\t"
- "mv %0, a0"
- : "=r"(ret), "+r"(a0), "+r"(a1)
- : "r"(a2), "r"(a3), "r"(a4), "r"(a7)
- : "cc", "memory");
- return ret;
-}
} // anonymous namespace
-#endif
\ No newline at end of file
+#endif /* LLVM_TOOLS_LLVM_BOLT_SYS_RISCV */
diff --git a/bolt/runtime/sys_x86_64.h b/bolt/runtime/sys_x86_64.h
index 933e939012481..18b46ead5559d 100644
--- a/bolt/runtime/sys_x86_64.h
+++ b/bolt/runtime/sys_x86_64.h
@@ -1,6 +1,16 @@
+//===- bolt/runtime/sys_x86_64.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
#ifndef LLVM_TOOLS_LLVM_BOLT_SYS_X86_64
#define LLVM_TOOLS_LLVM_BOLT_SYS_X86_64
+#include "runtime_types.h"
+
// Save all registers while keeping 16B stack alignment
#define SAVE_ALL \
"push %%rax\n" \
@@ -38,6 +48,45 @@
"pop %%rbx\n" \
"pop %%rax\n"
+#if defined(__APPLE__)
+#define __NR_exit 0x2000001
+#define __NR_read 0x2000003
+#define __NR_write 0x2000004
+#define __NR_sigprocmask 0x2000030
+#define __NR_munmap 0x2000049
+#define __NR_mmap 0x20000c5
+#define __NR_getpid 20
+#else
+// scripts/syscalltbl.sh --abis common,64 arch/x86/entry/syscalls/syscall_64.tbl
+// x86_64_syscalls.h
+#define __NR_read 0
+#define __NR_write 1
+#define __NR_close 3
+#define __NR_lseek 8
+#define __NR_mmap 9
+#define __NR_mprotect 10
+#define __NR_munmap 11
+#define __NR_rt_sigprocmask 14
+#define __NR_madvise 28
+#define __NR_nanosleep 35
+#define __NR_getpid 39
+#define __NR_clone 56
+#define __NR_fork 57
+#define __NR_exit 60
+#define __NR_kill 62
+#define __NR_uname 63
+#define __NR_fsync 74
+#define __NR_ftruncate 77
+#define __NR_setpgid 109
+#define __NR_getppid 110
+#define __NR_getpgid 121
+#define __NR_prctl 157
+#define __NR_getdents64 217
+#define __NR_exit_group 231
+#define __NR_openat 257
+#define __NR_readlinkat 267
+#endif
+
namespace {
// Get the difference between runtime address of .text section and
@@ -54,308 +103,6 @@ uint64_t getTextBaseAddress() {
return DynAddr - StaticAddr;
}
-#define _STRINGIFY(x) #x
-#define STRINGIFY(x) _STRINGIFY(x)
-
-uint64_t __read(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
-#if defined(__APPLE__)
-#define READ_SYSCALL 0x2000003
-#else
-#define READ_SYSCALL 0
-#endif
- __asm__ __volatile__("movq $" STRINGIFY(READ_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd), "S"(buf), "d"(count)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __write(uint64_t fd, const void *buf, uint64_t count) {
- uint64_t ret;
-#if defined(__APPLE__)
-#define WRITE_SYSCALL 0x2000004
-#else
-#define WRITE_SYSCALL 1
-#endif
- __asm__ __volatile__("movq $" STRINGIFY(WRITE_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd), "S"(buf), "d"(count)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-void *__mmap(uint64_t addr, uint64_t size, uint64_t prot, uint64_t flags,
- uint64_t fd, uint64_t offset) {
-#if defined(__APPLE__)
-#define MMAP_SYSCALL 0x20000c5
-#else
-#define MMAP_SYSCALL 9
-#endif
- void *ret;
- register uint64_t r8 asm("r8") = fd;
- register uint64_t r9 asm("r9") = offset;
- register uint64_t r10 asm("r10") = flags;
- __asm__ __volatile__("movq $" STRINGIFY(MMAP_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(addr), "S"(size), "d"(prot), "r"(r10), "r"(r8),
- "r"(r9)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __munmap(void *addr, uint64_t size) {
-#if defined(__APPLE__)
-#define MUNMAP_SYSCALL 0x2000049
-#else
-#define MUNMAP_SYSCALL 11
-#endif
- uint64_t ret;
- __asm__ __volatile__("movq $" STRINGIFY(MUNMAP_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(addr), "S"(size)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __sigprocmask(int how, const void *set, void *oldset) {
-#if defined(__APPLE__)
-#define SIGPROCMASK_SYSCALL 0x2000030
-#else
-#define SIGPROCMASK_SYSCALL 14
-#endif
- uint64_t ret;
- register long r10 asm("r10") = sizeof(uint64_t);
- __asm__ __volatile__("movq $" STRINGIFY(SIGPROCMASK_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(how), "S"(set), "d"(oldset), "r"(r10)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __getpid() {
- uint64_t ret;
-#if defined(__APPLE__)
-#define GETPID_SYSCALL 20
-#else
-#define GETPID_SYSCALL 39
-#endif
- __asm__ __volatile__("movq $" STRINGIFY(GETPID_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- :
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __exit(uint64_t code) {
-#if defined(__APPLE__)
-#define EXIT_SYSCALL 0x2000001
-#else
-#define EXIT_SYSCALL 231
-#endif
- uint64_t ret;
- __asm__ __volatile__("movq $" STRINGIFY(EXIT_SYSCALL) ", %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(code)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-#if !defined(__APPLE__)
-// We use a stack-allocated buffer for string manipulation in many pieces of
-// this code, including the code that prints each line of the fdata file. This
-// buffer needs to accommodate large function names, but shouldn't be
-// arbitrarily large (dynamically allocated) for simplicity of our memory space
-// usage.
-
-// Declare some syscall wrappers we use throughout this code to avoid linking
-// against system libc.
-uint64_t __open(const char *pathname, uint64_t flags, uint64_t mode) {
- uint64_t ret;
- __asm__ __volatile__("movq $2, %%rax\n"
- "syscall"
- : "=a"(ret)
- : "D"(pathname), "S"(flags), "d"(mode)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-long __getdents64(unsigned int fd, dirent64 *dirp, size_t count) {
- long ret;
- __asm__ __volatile__("movq $217, %%rax\n"
- "syscall"
- : "=a"(ret)
- : "D"(fd), "S"(dirp), "d"(count)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __readlink(const char *pathname, char *buf, size_t bufsize) {
- uint64_t ret;
- __asm__ __volatile__("movq $89, %%rax\n"
- "syscall"
- : "=a"(ret)
- : "D"(pathname), "S"(buf), "d"(bufsize)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __lseek(uint64_t fd, uint64_t pos, uint64_t whence) {
- uint64_t ret;
- __asm__ __volatile__("movq $8, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd), "S"(pos), "d"(whence)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __ftruncate(uint64_t fd, uint64_t length) {
- int ret;
- __asm__ __volatile__("movq $77, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd), "S"(length)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __close(uint64_t fd) {
- uint64_t ret;
- __asm__ __volatile__("movq $3, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __madvise(void *addr, size_t length, int advice) {
- int ret;
- __asm__ __volatile__("movq $28, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(addr), "S"(length), "d"(advice)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __uname(struct UtsNameTy *Buf) {
- int Ret;
- __asm__ __volatile__("movq $63, %%rax\n"
- "syscall\n"
- : "=a"(Ret)
- : "D"(Buf)
- : "cc", "rcx", "r11", "memory");
- return Ret;
-}
-
-uint64_t __nanosleep(const timespec *req, timespec *rem) {
- uint64_t ret;
- __asm__ __volatile__("movq $35, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(req), "S"(rem)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int64_t __fork() {
- uint64_t ret;
- __asm__ __volatile__("movq $57, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- :
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __mprotect(void *addr, size_t len, int prot) {
- int ret;
- __asm__ __volatile__("movq $10, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(addr), "S"(len), "d"(prot)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __getppid() {
- uint64_t ret;
- __asm__ __volatile__("movq $110, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- :
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __setpgid(uint64_t pid, uint64_t pgid) {
- int ret;
- __asm__ __volatile__("movq $109, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(pid), "S"(pgid)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-uint64_t __getpgid(uint64_t pid) {
- uint64_t ret;
- __asm__ __volatile__("movq $121, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(pid)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __kill(uint64_t pid, int sig) {
- int ret;
- __asm__ __volatile__("movq $62, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(pid), "S"(sig)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-int __fsync(int fd) {
- int ret;
- __asm__ __volatile__("movq $74, %%rax\n"
- "syscall\n"
- : "=a"(ret)
- : "D"(fd)
- : "cc", "rcx", "r11", "memory");
- return ret;
-}
-
-// %rdi %rsi %rdx %r10 %r8
-// sys_prctl int option unsigned unsigned unsigned unsigned
-// long arg2 long arg3 long arg4 long arg5
-int __prctl(int Option, unsigned long Arg2, unsigned long Arg3,
- unsigned long Arg4, unsigned long Arg5) {
- int Ret;
- register long rdx asm("rdx") = Arg3;
- register long r8 asm("r8") = Arg5;
- register long r10 asm("r10") = Arg4;
- __asm__ __volatile__("movq $157, %%rax\n"
- "syscall\n"
- : "=a"(Ret)
- : "D"(Option), "S"(Arg2), "d"(rdx), "r"(r10), "r"(r8)
- :);
- return Ret;
-}
-
-#endif
-
} // anonymous namespace
-#endif
+#endif /* LLVM_TOOLS_LLVM_BOLT_SYS_X86_64 */
diff --git a/bolt/runtime/syscall_wrappers.h b/bolt/runtime/syscall_wrappers.h
new file mode 100644
index 0000000000000..5e7711ed1dadf
--- /dev/null
+++ b/bolt/runtime/syscall_wrappers.h
@@ -0,0 +1,208 @@
+//===- bolt/runtime/syscall_wrappers.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_BOLT_SYSCALL_WRAPPERS
+#define LLVM_TOOLS_LLVM_BOLT_SYSCALL_WRAPPERS
+
+#if defined(__aarch64__) || defined(__arm64__)
+#include "sys_aarch64.h"
+#elif defined(__riscv)
+#include "sys_riscv64.h"
+#elif defined(__x86_64__)
+#include "sys_x86_64.h"
+#else
+#error "For AArch64/ARM64,X86_64 AND RISCV64 only."
+#endif
+
+#include "runtime_types.h"
+
+// LLVM's libc syscall wrappers
+#define LIBC_NAMESPACE __bolt_runtime_syscall_wrappres
+#include "src/__support/OSUtil/linux/syscall.h"
+
+// Anonymous namespace covering everything but our library entry point
+namespace {
+
+// Declare some syscall wrappers we use throughout this code to avoid linking
+// against system libc.
+// Syscall wrapper return and argument types are based on LLVM's libc.
+
+// Reference: libc/src/unistd/linux/read.cpp
+ssize_t __read(int fd, void *buf, size_t count) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<ssize_t>(__NR_read, fd, buf, count);
+}
+
+// Reference: libc/src/unistd/linux/write.cpp
+ssize_t __write(int fd, const void *buf, size_t count) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<ssize_t>(__NR_write, fd, buf, count);
+}
+
+// Reference: libc/src/sys/mman/linux/mmap.cpp
+void *__mmap(void *addr, size_t size, int prot, int flags, int fd,
+ off_t offset) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<void *>(__NR_mmap, addr, size, prot,
+ flags, fd, offset);
+}
+
+// Reference: libc/src/sys/mman/linux/munmap.cpp
+int __munmap(void *addr, size_t size) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_munmap, addr, size);
+}
+
+// Reference: libc/src/signal/linux/sigprocmask.cpp
+int __sigprocmask(int how, const sigset_t *set, sigset_t *oldset) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_rt_sigprocmask, how, set,
+ oldset, sizeof(sigset_t));
+}
+
+// Reference: libc/src/unistd/linux/getpid.cpp
+pid_t __getpid() {
+ return LIBC_NAMESPACE_DECL::syscall_impl<pid_t>(__NR_getpid);
+}
+
+// Reference: libc/src/__support/OSUtil/linux/exit.cpp
+#if defined(__x86_64__)
+__attribute__((no_stack_protector))
+#endif
+__attribute__((noreturn)) void __exit(int status) {
+ for (;;) {
+ LIBC_NAMESPACE_DECL::syscall_impl<long>(__NR_exit_group, status);
+ LIBC_NAMESPACE_DECL::syscall_impl<long>(__NR_exit, status);
+ }
+}
+
+#if !defined(__APPLE__)
+
+// Reference: libc/src/fcntl/linux/open.cpp
+int __open(const char *path, int flags, mode_t mode) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_openat, AT_FDCWD, path,
+ flags, mode);
+}
+
+// Reference: libc/src/unistd/linux/readlink.cpp
+ssize_t __readlink(const char *path, char *buf, size_t bufsize) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<ssize_t>(__NR_readlinkat, AT_FDCWD,
+ path, buf, bufsize);
+}
+
+// Reference: libc/src/unistd/linux/lseek.cpp
+off_t __lseek(int fd, off_t offset, int whence) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<off_t>(__NR_lseek, fd, offset,
+ whence);
+}
+
+// Reference: libc/src/unistd/linux/ftruncate.cpp
+int __ftruncate(int fd, off_t len) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_ftruncate, fd, len);
+}
+
+// Reference: libc/src/unistd/linux/close.cpp
+int __close(int fd) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_close, fd);
+}
+
+// Reference: libc/src/sys/utsname/linux/uname.cpp
+int __uname(struct UtsNameTy *name) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_uname, name);
+}
+
+// Reference: libc/src/sys/mman/linux/mprotect.cpp
+int __mprotect(void *start, size_t size, int prot) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_mprotect, start, size,
+ prot);
+}
+
+// Reference: libc/src/signal/linux/kill.cpp
+int __kill(pid_t pid, int sig) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_kill, pid, sig);
+}
+
+// Reference: libc/src/unistd/linux/fsync.cpp
+int __fsync(int fd) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_fsync, fd);
+}
+
+// Reference: libc/src/sys/prctl/linux/prctl.cpp
+int __prctl(int option, unsigned long arg2, unsigned long arg3,
+ unsigned long arg4, unsigned long arg5) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_prctl, option, arg2, arg3,
+ arg4, arg5);
+}
+
+// Reference: libc/src/sys/mman/linux/madvise.cpp
+int __madvise(void *addr, size_t size, int advice) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_madvise, addr, size,
+ advice);
+}
+
+// Reference: libc/src/time/linux/nanosleep.cpp
+int __nanosleep(const timespec *req, timespec *rem) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_nanosleep, req, rem);
+}
+
+// FIXME: cleanup here
+// Currently, the arch-specific implementations have not been changed in order
+// to avoid breaking the instrumentation functionality:
+// - x64_64 - fork syscall
+// - aarch64/riscv64 - clone syscall
+// arm64/riscv64 selects CONFIG_CLONE_BACKWARDS (child_tid 5th arg in sys_clone)
+// but x86_64 doesn't (child_tid 4th arg).
+// x86_64: https://github.com/torvalds/linux/blob/v7.1/kernel/fork.c#L2847
+// arch64/riscv64:
+// https://github.com/torvalds/linux/blob/v7.1/kernel/fork.c#L2831
+// Not clear why CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID. In this case
+// child_tid should be set.
+// Reference (glibc): sysdeps/unix/sysv/linux/arch-fork.h
+// Reference: libc/src/unistd/linux/fork.cpp
+pid_t __fork() {
+#if defined(__x86_64__)
+ return LIBC_NAMESPACE_DECL::syscall_impl<pid_t>(__NR_fork);
+#elif defined(__aarch64__) || defined(__arm64__) || defined(__riscv)
+ return LIBC_NAMESPACE_DECL::syscall_impl<pid_t>(
+ __NR_clone, CLONE_CHILD_CLEARTID | CLONE_CHILD_SETTID | SIGCHLD, 0, 0, 0,
+ 0);
+#endif
+}
+
+// Reference: libc/src/__support/File/linux/dir.cpp
+ssize_t __getdents64(unsigned int fd, dirent64 *dirp, unsigned int count) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<ssize_t>(__NR_getdents64, fd, dirp,
+ count);
+}
+
+// Reference: libc/src/unistd/linux/getppid.cpp
+pid_t __getppid() {
+ return LIBC_NAMESPACE_DECL::syscall_impl<pid_t>(__NR_getppid);
+}
+
+// No reference information was found in LLVM's libc; glibc and the Linux kernel
+// were used instead.
+
+// Reference
+// - glibc: posix/getpgid.c
+// pid_t __getpgid (pid_t pid)
+// - Linux kernel: kernel/sys.c
+// SYSCALL_DEFINE1(getpgid, pid_t, pid)
+pid_t __getpgid(pid_t pid) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<pid_t>(__NR_getpgid, pid);
+}
+
+// Reference
+// - glibc: posix/setpgid.c
+// int __setpgid (int pid, int pgid)
+// - Linux kernel: kernel/sys.c
+// SYSCALL_DEFINE2(setpgid, pid_t, pid, pid_t, pgid)
+int __setpgid(pid_t pid, pid_t pgid) {
+ return LIBC_NAMESPACE_DECL::syscall_impl<int>(__NR_setpgid, pid, pgid);
+}
+
+#endif
+
+} // anonymous namespace
+
+#endif /* LLVM_TOOLS_LLVM_BOLT_SYSCALL_WRAPPERS */
More information about the llvm-commits
mailing list