[flang-commits] [clang] [flang] [llvm] [flang] Add runtime trampoline pool for W^X compliance (PR #183108)
Eugene Epshteyn via flang-commits
flang-commits at lists.llvm.org
Sun Mar 1 06:49:24 PST 2026
================
@@ -0,0 +1,425 @@
+//===-- lib/runtime/trampoline.cpp -------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// W^X-compliant trampoline pool implementation.
+//
+// This file implements a runtime trampoline pool that maintains separate
+// memory regions for executable code (RX) and writable data (RW).
+//
+// On Linux the code region transitions RW → RX (never simultaneously W+X).
+// On macOS Apple Silicon the code region uses MAP_JIT with per-thread W^X
+// toggling via pthread_jit_write_protect_np, so the mapping permissions
+// include both W and X but hardware enforces that only one is active at
+// a time on any given thread.
+//
+// Architecture:
+// - Code region (RX): Contains pre-assembled trampoline stubs that load
+// callee address and static chain from a paired TDATA entry, then jump
+// to the callee with the static chain in the appropriate register.
+// - Data region (RW): Contains TrampolineData entries with {callee_address,
+// static_chain_address} pairs, one per trampoline slot.
+// - Free list: Tracks available trampoline slots for O(1) alloc/free.
+//
+// Thread safety: Uses Fortran::runtime::Lock (pthreads on POSIX,
+// CRITICAL_SECTION on Windows) — not std::mutex — to avoid C++ runtime
+// library dependence. A single global lock serializes pool operations.
+// This is a deliberate V1 design choice to keep the initial W^X
+// architectural change minimal. Per-thread lock-free pools are deferred
+// to a future optimization patch.
+//
+// AddressSanitizer note: The trampoline code region is allocated via
+// mmap (not malloc/new), so ASan does not track it. The data region
+// and handles are allocated via malloc (through AllocateMemoryOrCrash),
+// which ASan intercepts normally. No special annotations are needed.
+//
+// See flang/docs/InternalProcedureTrampolines.md for design details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Runtime/trampoline.h"
+#include "flang-rt/runtime/lock.h"
+#include "flang-rt/runtime/memory.h"
+#include "flang-rt/runtime/terminator.h"
+#include "flang-rt/runtime/trampoline.h"
+
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <new> // For placement-new only (no operator new/delete dependency)
+
+// Platform-specific headers for memory mapping.
+#if defined(_WIN32)
+#include <windows.h>
+#else
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
+
+// macOS Apple Silicon requires MAP_JIT and pthread_jit_write_protect_np
+// to create executable memory under the hardened runtime.
+#if defined(__APPLE__) && defined(__aarch64__)
+#include <libkern/OSCacheControl.h>
+#include <pthread.h>
+#endif
+
+// Architecture support check. Stub generators exist only for x86-64 and
+// AArch64. On other architectures the file compiles but the runtime API
+// functions crash with a diagnostic if actually called, so that building
+// flang-rt on e.g. RISC-V or PPC64 never fails.
+#if defined(__x86_64__) || defined(_M_X64) || defined(__aarch64__) || \
+ defined(_M_ARM64)
+#define TRAMPOLINE_ARCH_SUPPORTED 1
+#else
+#define TRAMPOLINE_ARCH_SUPPORTED 0
+#endif
+
+namespace Fortran::runtime::trampoline {
+
+/// A handle returned to the caller. Contains enough info to find
+/// both the trampoline stub and its data entry.
+struct TrampolineHandle {
+ void *codePtr; // Pointer to the trampoline stub in the RX region.
+ TrampolineData *dataPtr; // Pointer to the data entry in the RW region.
+ std::size_t slotIndex; // Index in the pool for free-list management.
+};
+
+// Namespace-scope globals following Flang runtime conventions:
+// - Lock is trivially constructible (pthread_mutex_t / CRITICAL_SECTION)
+// - Pool pointer starts null; initialized under lock (double-checked locking)
+class TrampolinePool; // Forward declaration for pointer below.
+static Lock poolLock;
+static TrampolinePool *poolInstance{nullptr};
+
+/// The global trampoline pool.
+class TrampolinePool {
+public:
+ static TrampolinePool &instance() {
+ if (poolInstance) {
+ return *poolInstance;
+ }
+ CriticalSection critical{poolLock};
+ if (poolInstance) {
+ return *poolInstance;
+ }
+ // Allocate pool using malloc + placement new (trivial constructor).
+ Terminator terminator{__FILE__, __LINE__};
+ void *storage = AllocateMemoryOrCrash(terminator, sizeof(TrampolinePool));
+ poolInstance = new (storage) TrampolinePool();
+ return *poolInstance;
+ }
+
+ /// Allocate a trampoline slot and initialize it.
+ TrampolineHandle *allocate(
+ const void *calleeAddress, const void *staticChainAddress) {
+ CriticalSection critical{lock_};
+ ensureInitialized();
+
+ if (freeHead_ == kInvalidIndex) {
+ // Pool exhausted — fixed size by design for V1.
+ // The pool capacity is controlled by FLANG_TRAMPOLINE_POOL_SIZE
+ // (default 1024). Dynamic slab growth can be added in a follow-up
+ // patch if real workloads demonstrate a need for it.
+ Terminator terminator{__FILE__, __LINE__};
+ terminator.Crash("Trampoline pool exhausted (max %zu slots). "
+ "Set FLANG_TRAMPOLINE_POOL_SIZE to increase.",
+ poolSize_);
+ }
+
+ std::size_t index = freeHead_;
+ freeHead_ = freeList_[index];
+
+ // Initialize the data entry.
+ dataRegion_[index].calleeAddress = calleeAddress;
+ dataRegion_[index].staticChainAddress = staticChainAddress;
+
+ // Create handle using malloc + placement new.
+ Terminator terminator{__FILE__, __LINE__};
+ void *mem = AllocateMemoryOrCrash(terminator, sizeof(TrampolineHandle));
+ auto *handle = new (mem) TrampolineHandle();
+ handle->codePtr =
+ static_cast<char *>(codeRegion_) + index * kTrampolineStubSize;
+ handle->dataPtr = &dataRegion_[index];
+ handle->slotIndex = index;
+
+ return handle;
+ }
+
+ /// Get the callable address of a trampoline.
+ void *getCallableAddress(TrampolineHandle *handle) { return handle->codePtr; }
+
+ /// Free a trampoline slot.
+ void free(TrampolineHandle *handle) {
+ CriticalSection critical{lock_};
+
+ std::size_t index = handle->slotIndex;
+
+ // Poison the data entry so that any dangling call through a freed
+ // trampoline traps immediately. We use a non-null, obviously-invalid
+ // address (0xDEAD...) so that the resulting fault is distinguishable
+ // from a null-pointer dereference when debugging.
+ dataRegion_[index].calleeAddress = reinterpret_cast<const void *>(
+ static_cast<uintptr_t>(~uintptr_t{0} - 1));
+ dataRegion_[index].staticChainAddress = nullptr;
+
+ // Return slot to free list.
+ freeList_[index] = freeHead_;
+ freeHead_ = index;
+
+ FreeMemory(handle);
+ }
+
+private:
+ static constexpr std::size_t kInvalidIndex = ~std::size_t{0};
+
+ TrampolinePool() = default;
+
+ void ensureInitialized() {
+ if (initialized_)
+ return;
+ initialized_ = true;
+
+ // Check environment variable for pool size override.
+ // Fixed-size pool by design (V1): avoids complexity of dynamic growth
+ // and re-protection of code pages. The default (1024 slots) is
+ // sufficient for typical Fortran programs. Users can override via:
+ // export FLANG_TRAMPOLINE_POOL_SIZE=4096
+ poolSize_ = kDefaultPoolSize;
+ if (const char *envSize = std::getenv("FLANG_TRAMPOLINE_POOL_SIZE")) {
+ long val = std::strtol(envSize, nullptr, 10);
+ if (val > 0)
+ poolSize_ = static_cast<std::size_t>(val);
+ }
+
+ // Allocate the data region (RW).
+ dataRegion_ = static_cast<TrampolineData *>(
+ std::calloc(poolSize_, sizeof(TrampolineData)));
+ assert(dataRegion_ && "Failed to allocate trampoline data region");
+
+ // Allocate the code region (initially RW for writing stubs, then RX).
+ std::size_t codeSize = poolSize_ * kTrampolineStubSize;
+#if defined(_WIN32)
+ codeRegion_ = VirtualAlloc(
+ nullptr, codeSize, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
+#elif defined(__APPLE__) && defined(__aarch64__)
+ // macOS Apple Silicon: MAP_JIT is required for pages that will become
+ // executable. Use pthread_jit_write_protect_np to toggle W↔X.
+ codeRegion_ = mmap(nullptr, codeSize, PROT_READ | PROT_WRITE | PROT_EXEC,
+ MAP_PRIVATE | MAP_ANONYMOUS | MAP_JIT, -1, 0);
+ if (codeRegion_ == MAP_FAILED)
+ codeRegion_ = nullptr;
+ if (codeRegion_) {
+ // Enable writing on this thread (MAP_JIT defaults to execute).
+ pthread_jit_write_protect_np(0); // 0 = writable
+ }
+#else
+ codeRegion_ = mmap(nullptr, codeSize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (codeRegion_ == MAP_FAILED)
+ codeRegion_ = nullptr;
+#endif
+ assert(codeRegion_ && "Failed to allocate trampoline code region");
+
+ // Generate trampoline stubs.
+ generateStubs();
+
+ // Flush instruction cache. Required on architectures with non-coherent
+ // I-cache/D-cache (AArch64, PPC, etc.). On x86-64 this is a no-op
+ // but harmless. Without this, AArch64 may execute stale instructions.
+#if defined(__APPLE__) && defined(__aarch64__)
+ // On macOS, use sys_icache_invalidate (from libkern/OSCacheControl.h).
+ sys_icache_invalidate(codeRegion_, codeSize);
+#elif defined(_WIN32)
+ FlushInstructionCache(GetCurrentProcess(), codeRegion_, codeSize);
+#else
+ __builtin___clear_cache(static_cast<char *>(codeRegion_),
+ static_cast<char *>(codeRegion_) + codeSize);
+#endif
+
+ // Make code region executable and non-writable (W^X).
+#if defined(_WIN32)
+ DWORD oldProtect;
+ VirtualProtect(codeRegion_, codeSize, PAGE_EXECUTE_READ, &oldProtect);
+#elif defined(__APPLE__) && defined(__aarch64__)
+ // Switch back to execute-only (MAP_JIT manages per-thread W^X).
+ pthread_jit_write_protect_np(1); // 1 = executable
+#else
+ mprotect(codeRegion_, codeSize, PROT_READ | PROT_EXEC);
+#endif
+
+ // Initialize free list.
+ freeList_ = static_cast<std::size_t *>(
+ std::malloc(poolSize_ * sizeof(std::size_t)));
+ assert(freeList_ && "Failed to allocate trampoline free list");
+
+ for (std::size_t i = 0; i < poolSize_ - 1; ++i)
+ freeList_[i] = i + 1;
+ freeList_[poolSize_ - 1] = kInvalidIndex;
+ freeHead_ = 0;
+ }
+
+ /// Generate platform-specific trampoline stubs in the code region.
+ /// Each stub loads callee address and static chain from its paired
+ /// TDATA entry and jumps to the callee.
+ void generateStubs() {
+#if defined(__x86_64__) || defined(_M_X64)
+ generateStubsX86_64();
+#elif defined(__aarch64__) || defined(_M_ARM64)
+ generateStubsAArch64();
+#else
+ // Unsupported architecture — should never be reached because the
+ // extern "C" API functions guard with TRAMPOLINE_ARCH_SUPPORTED.
+ // Fill with trap bytes as a safety net.
+ std::memset(codeRegion_, 0, poolSize_ * kTrampolineStubSize);
+#endif
+ }
+
+#if defined(__x86_64__) || defined(_M_X64)
+ /// Generate x86-64 trampoline stubs.
+ ///
+ /// Each stub does:
+ /// movabsq $dataEntry, %r11 ; load TDATA entry address
+ /// movq 8(%r11), %r10 ; load static chain -> nest register
+ /// jmpq *(%r11) ; jump to callee address
+ ///
+ /// Total: 10 + 4 + 3 = 17 bytes, padded to kTrampolineStubSize.
+ void generateStubsX86_64() {
+ auto *code = static_cast<uint8_t *>(codeRegion_);
+
+ for (std::size_t i = 0; i < poolSize_; ++i) {
+ uint8_t *stub = code + i * kTrampolineStubSize;
+
+ // Address of the corresponding TDATA entry.
+ auto dataAddr = reinterpret_cast<uint64_t>(&dataRegion_[i]);
+
+ std::size_t off = 0;
+
+ // movabsq $dataAddr, %r11 (REX.W + B, opcode 0xBB for r11)
+ stub[off++] = 0x49; // REX.WB
+ stub[off++] = 0xBB; // MOV r11, imm64
+ std::memcpy(&stub[off], &dataAddr, 8);
+ off += 8;
+
+ // movq 8(%r11), %r10 (load staticChainAddress into r10)
+ stub[off++] = 0x4D; // REX.WRB
+ stub[off++] = 0x8B; // MOV r/m64 -> r64
+ stub[off++] = 0x53; // ModRM: [r11 + disp8], r10
+ stub[off++] = 0x08; // disp8 = 8
+
+ // jmpq *(%r11) (jump to calleeAddress)
+ stub[off++] = 0x41; // REX.B
+ stub[off++] = 0xFF; // JMP r/m64
+ stub[off++] = 0x23; // ModRM: [r11], opcode extension 4
+
+ // Pad the rest with INT3 (0xCC) for safety.
+ while (off < kTrampolineStubSize)
+ stub[off++] = 0xCC;
+ }
+ }
+#endif
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+ /// Generate AArch64 trampoline stubs.
+ ///
+ /// Each stub does:
----------------
eugeneepshteyn wrote:
AI approved of your changes :-)
https://github.com/llvm/llvm-project/pull/183108
More information about the flang-commits
mailing list