[Mlir-commits] [llvm] [mlir] [AArch64][RuntimeDyld] Handle far external symbols in COFF JIT code (PR #229719)
llvmlistbot at llvm.org
llvmlistbot at llvm.org
Wed Oct 7 05:06:46 PDT 2026
https://github.com/yafet-a updated https://github.com/llvm/llvm-project/pull/229719
>From 02bc49045a15f3f57d1b055df95ddadd00edb6c9 Mon Sep 17 00:00:00 2001
From: Reid Kleckner <rkleckner at nvidia.com>
Date: Tue, 22 Sep 2026 18:05:24 +0000
Subject: [PATCH] [AArch64][RuntimeDyld] Handle far external symbols in COFF
JIT code
External symbols on Windows can be more than 4 GiB from JIT allocations and
therefore outside ADRP's signed 21-bit page range. Detect ADRP relocation
overflow instead of silently truncating it.
For an external address materialized with an ADRP+ADD pair, create a pointer
slot in the section's stub area and rewrite the ADD to an LDR from that slot.
Leave non-zero addends and other instruction patterns unchanged.
Add llvm-rtdyld coverage and a Windows-on-Arm MLIR ExecutionEngine test that
places JIT allocations more than 4 GiB from external symbols.
---
.../Targets/RuntimeDyldCOFFAArch64.h | 93 ++++++++++
.../RuntimeDyld/AArch64/COFF_AArch64.s | 31 +++-
mlir/unittests/ExecutionEngine/CMakeLists.txt | 1 +
.../ExecutionEngine/WindowsArm64.cpp | 175 ++++++++++++++++++
4 files changed, 299 insertions(+), 1 deletion(-)
create mode 100644 mlir/unittests/ExecutionEngine/WindowsArm64.cpp
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
index 3ca6309874de5..bf8c8667a1140 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h
@@ -66,6 +66,15 @@ static void write32AArch64Addr(void *T, uint64_t s, uint64_t p, int shift) {
using namespace llvm::support::endian;
uint64_t Imm = (s >> shift) - (p >> shift);
+ // ADRP's page offset is 21 bits signed (+/-4 GiB); fail loudly rather than
+ // truncate into a wild pointer.
+ if (!isInt<21>(static_cast<int64_t>(Imm))) {
+ report_fatal_error(Twine("ADRP target out of range: target=") + Twine(s) +
+ " pc=" + Twine(p) +
+ " delta=" + Twine(s > p ? s - p : p - s) +
+ (s > p ? " (target above pc)" : " (target below pc)") +
+ " -- needs a 21-bit signed page offset (+/-4 GiB)");
+ }
uint32_t ImmLo = (Imm & 0x3) << 29;
uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;
uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);
@@ -108,6 +117,48 @@ class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF {
unsigned getMaxStubSize() const override { return 20; }
+ /// GOT slot in this section's stub area holding the address of \p Name, so
+ /// an ADRP can reach external symbols (e.g. DLL exports) beyond +/-4 GiB.
+ /// computeSectionStubBufSize reserves getMaxStubSize() per relocation, which
+ /// covers the 8 bytes.
+ uint64_t getGOTEntryOffset(unsigned SectionID, StubMap &Stubs, StringRef Name,
+ bool CreateIfMissing) {
+ // Key on the symbol alone; ~0 cannot collide with a branch stub's offset.
+ RelocationValueRef Key;
+ Key.SectionID = SectionID;
+ Key.Offset = ~uint64_t(0);
+ Key.SymbolName = Name.data();
+
+ // RelocationValueRef compares symbol-name pointers rather than their
+ // contents, but paired COFF relocations need not carry the same pointer.
+ for (const auto &[StubKey, StubOffset] : Stubs)
+ if (StubKey.SectionID == SectionID && StubKey.Offset == ~uint64_t(0) &&
+ StubKey.SymbolName && StringRef(StubKey.SymbolName) == Name)
+ return StubOffset;
+ if (!CreateIfMissing)
+ return ~uint64_t(0);
+
+ SectionEntry &Sec = Sections[SectionID];
+ uint64_t EntryOffset = alignTo(Sec.getStubOffset(), 8);
+ Sec.advanceStubOffset(EntryOffset + 8 - Sec.getStubOffset());
+ Stubs[Key] = EntryOffset;
+
+ // Have the resolver write the symbol's absolute address into the slot.
+ RelocationEntry RE(SectionID, EntryOffset, COFF::IMAGE_REL_ARM64_ADDR64, 0,
+ /*IsPCRel=*/false, Log2_64(8));
+ addRelocationForSymbol(RE, Name);
+
+ LLVM_DEBUG(dbgs() << " Created GOT entry for " << Name << " at offset "
+ << EntryOffset << "\n");
+ return EntryOffset;
+ }
+
+ /// ADD (immediate, no shift): the second half of the ADRP+ADD idiom.
+ static bool isAddImmediate(uint32_t Inst) {
+ // sf 0 0 100010 sh imm12 Rn Rd with sh == 0; excludes ADDS (bit 29).
+ return (Inst & 0x7FC00000) == 0x11000000;
+ }
+
std::tuple<uint64_t, uint64_t, uint64_t>
generateRelocationStub(unsigned SectionID, StringRef TargetName,
uint64_t Offset, uint64_t RelType, uint64_t Addend,
@@ -231,14 +282,56 @@ class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF {
case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: {
uint32_t orig = read32le(Displacement);
Addend = ((orig >> 29) & 0x3) | ((orig >> 3) & 0x1FFFFC);
+
+ // External symbols are routinely beyond ADRP's +/-4 GiB on Windows and
+ // there is no veneer, so route ADRP+ADD through a GOT slot. Only that
+ // idiom qualifies: ADRP+LDR reads the symbol's contents, and a non-zero
+ // addend cannot be expressed by a slot holding plain `sym`. LLVM codegen
+ // emits direct addresses as a MOVaddr pseudo and expands it after
+ // scheduling into this adjacent ADRP+ADD pair.
+ if (IsExtern && Addend == 0 &&
+ RelType == COFF::IMAGE_REL_ARM64_PAGEBASE_REL21) {
+ uint32_t NextInst = read32le(Displacement + 4);
+ if (isAddImmediate(NextInst) && ((NextInst >> 10) & 0xFFF) == 0) {
+ TargetOffset = getGOTEntryOffset(SectionID, Stubs, TargetName,
+ /*CreateIfMissing=*/true);
+ TargetSectionID = SectionID;
+ TargetName = StringRef();
+ IsExtern = false;
+ Addend = 0;
+ }
+ }
break;
}
case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L:
case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: {
uint32_t orig = read32le(Displacement);
Addend = ((orig >> 10) & 0xFFF);
+
+ // Partner of the ADRP above; no slot means that ADRP was left alone.
+ if (IsExtern && Addend == 0 &&
+ RelType == COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A) {
+ uint64_t GOTOffset = getGOTEntryOffset(SectionID, Stubs, TargetName,
+ /*CreateIfMissing=*/false);
+ if (GOTOffset != ~uint64_t(0) && isAddImmediate(orig)) {
+ // ADD Xd, Xn, #imm -> LDR Xd, [Xn, #imm]: load the slot instead.
+ uint32_t Rd = orig & 0x1F;
+ uint32_t Rn = (orig >> 5) & 0x1F;
+ write32le(AddendSection.getAddressWithOffset(Offset),
+ 0xF9400000u | (Rn << 5) | Rd);
+
+ // 12L so the resolver scales the immediate for a 64-bit load.
+ RelType = COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L;
+ TargetOffset = GOTOffset;
+ TargetSectionID = SectionID;
+ TargetName = StringRef();
+ IsExtern = false;
+ Addend = 0;
+ }
+ }
break;
}
+
case COFF::IMAGE_REL_ARM64_ADDR64: {
Addend = read64le(Displacement);
break;
diff --git a/llvm/test/ExecutionEngine/RuntimeDyld/AArch64/COFF_AArch64.s b/llvm/test/ExecutionEngine/RuntimeDyld/AArch64/COFF_AArch64.s
index 9a31952b69973..57ad249048ac4 100644
--- a/llvm/test/ExecutionEngine/RuntimeDyld/AArch64/COFF_AArch64.s
+++ b/llvm/test/ExecutionEngine/RuntimeDyld/AArch64/COFF_AArch64.s
@@ -1,5 +1,6 @@
+
// RUN: llvm-mc -triple aarch64-windows -filetype obj -o %t.obj %s
-// RUN: llvm-rtdyld -triple aarch64-windows -dummy-extern dummy=0x79563413 -dummy-extern dummyA=0x78566879 -target-addr-start=40960000000000 -verify -check %s %t.obj
+// RUN: llvm-rtdyld -triple aarch64-windows -dummy-extern dummy=0x79563413 -dummy-extern dummyA=0x78566879 -dummy-extern far_data=0x1122334455667788 -dummy-extern near_data=40960000001000 -target-addr-start=40960000000000 -verify -check %s %t.obj
.text
.def _bnamed
@@ -80,6 +81,34 @@ ldr1:
ldr x0, [x0, #:lo12:_const]
ret
+# A far external address formed with ADRP+ADD is rewritten to ADRP+LDR through
+# a pointer slot in this section's stub area.
+# rtdyld-check: decode_operand(far_adrp, 1) = (stub_addr(COFF_AArch64.s.tmp.obj/.text, far_data)[32:12] - far_adrp[32:12])
+# rtdyld-check: (*{4}far_ldr)[31:22] = 0x3e5
+# rtdyld-check: decode_operand(far_ldr, 2) = stub_addr(COFF_AArch64.s.tmp.obj/.text, far_data)[11:3]
+# rtdyld-check: *{8}stub_addr(COFF_AArch64.s.tmp.obj/.text, far_data) = 0x1122334455667788
+ .globl _test_far_external_address
+ .align 2
+_test_far_external_address:
+far_adrp:
+ adrp x0, far_data
+far_ldr:
+ add x0, x0, far_data at PAGEOFF
+ ret
+
+# A non-zero page-offset addend cannot use a slot holding plain `sym`. Keep the
+# original ADRP+ADD pair instead of partially rewriting it to address the slot.
+# rtdyld-check: (*{4}near_addend_add)[31:22] = 0x244
+# rtdyld-check: decode_operand(near_addend_add, 2) = (near_data+8)[11:0]
+ .globl _test_external_address_with_addend
+ .align 2
+_test_external_address_with_addend:
+ adrp x0, near_data
+near_addend_add:
+ add x0, x0, near_data at PAGEOFF+8
+ ret
+
+
.globl _test_add_reloc
.align 2
diff --git a/mlir/unittests/ExecutionEngine/CMakeLists.txt b/mlir/unittests/ExecutionEngine/CMakeLists.txt
index b83163e39c014..e1b3a79dd939f 100644
--- a/mlir/unittests/ExecutionEngine/CMakeLists.txt
+++ b/mlir/unittests/ExecutionEngine/CMakeLists.txt
@@ -9,6 +9,7 @@ add_mlir_unittest(MLIRExecutionEngineTests
DynamicMemRef.cpp
StridedMemRef.cpp
Invoke.cpp
+ WindowsArm64.cpp
)
mlir_target_link_libraries(MLIRExecutionEngineTests
diff --git a/mlir/unittests/ExecutionEngine/WindowsArm64.cpp b/mlir/unittests/ExecutionEngine/WindowsArm64.cpp
new file mode 100644
index 0000000000000..be6adf8aede2d
--- /dev/null
+++ b/mlir/unittests/ExecutionEngine/WindowsArm64.cpp
@@ -0,0 +1,175 @@
+//===- WindowsArm64.cpp - Windows on Arm execution engine tests -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#if defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__))
+
+#include "mlir/ExecutionEngine/ExecutionEngine.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/MLIRContext.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+#include "gtest/gtest.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <limits>
+#include <system_error>
+
+using namespace mlir;
+
+namespace {
+constexpr uintptr_t FourGiB = uintptr_t{4} << 30;
+constexpr uintptr_t EightGiB = uintptr_t{8} << 30;
+
+static int FarData;
+static int CtorCount;
+static int DtorCount;
+
+static void ctorCallback() { ++CtorCount; }
+static void dtorCallback() { ++DtorCount; }
+
+// Places JIT allocations at least 8 GiB from Anchor. This makes the test
+// independent of Windows ASLR's usual (but not guaranteed) separation of the
+// module-image, heap, and VirtualAlloc regions.
+class FarMemoryMapper final : public llvm::SectionMemoryManager::MemoryMapper {
+public:
+ explicit FarMemoryMapper(uintptr_t Anchor)
+ : Anchor(Anchor),
+ NextHint(Anchor > EightGiB ? Anchor - EightGiB : Anchor + EightGiB) {}
+
+ llvm::sys::MemoryBlock
+ allocateMappedMemory(llvm::SectionMemoryManager::AllocationPurpose,
+ size_t NumBytes, const llvm::sys::MemoryBlock *,
+ unsigned Flags, std::error_code &EC) override {
+ // LLVM's Windows mapper passes the end of Near as a VirtualAlloc address
+ // hint. A zero-sized fake block therefore gives this test direct control
+ // of the requested address without committing a multi-gigabyte range.
+ llvm::sys::MemoryBlock Hint(reinterpret_cast<void *>(NextHint), 0);
+ llvm::sys::MemoryBlock Block =
+ llvm::sys::Memory::allocateMappedMemory(NumBytes, &Hint, Flags, EC);
+ if (EC)
+ return Block;
+
+ uintptr_t Address = reinterpret_cast<uintptr_t>(Block.base());
+ uintptr_t Distance = Address > Anchor ? Address - Anchor : Anchor - Address;
+ if (Distance <= FourGiB) {
+ llvm::sys::Memory::releaseMappedMemory(Block);
+ EC = std::make_error_code(std::errc::not_enough_memory);
+ return {};
+ }
+
+ MinimumDistance = std::min(MinimumDistance, Distance);
+ // Keep all JIT objects close to each other while preserving their distance
+ // from the external symbols in this executable.
+ NextHint = Address + Block.allocatedSize();
+ return Block;
+ }
+
+ std::error_code protectMappedMemory(const llvm::sys::MemoryBlock &Block,
+ unsigned Flags) override {
+ return llvm::sys::Memory::protectMappedMemory(Block, Flags);
+ }
+
+ std::error_code releaseMappedMemory(llvm::sys::MemoryBlock &Block) override {
+ return llvm::sys::Memory::releaseMappedMemory(Block);
+ }
+
+ uintptr_t getMinimumDistance() const { return MinimumDistance; }
+
+private:
+ uintptr_t Anchor;
+ uintptr_t NextHint;
+ uintptr_t MinimumDistance = std::numeric_limits<uintptr_t>::max();
+};
+
+static std::unique_ptr<llvm::Module>
+buildFarAddressModule(Operation *, llvm::LLVMContext &context) {
+ auto module = std::make_unique<llvm::Module>("windows-arm64-far", context);
+ llvm::IRBuilder<> builder(context);
+ auto *voidFnTy = llvm::FunctionType::get(builder.getVoidTy(), false);
+
+ auto addCtorOrDtor = [&](llvm::StringRef callbackName,
+ llvm::StringRef functionName, bool isCtor) {
+ auto *callback = llvm::Function::Create(
+ voidFnTy, llvm::GlobalValue::ExternalLinkage, callbackName, *module);
+ auto *function = llvm::Function::Create(
+ voidFnTy, llvm::GlobalValue::InternalLinkage, functionName, *module);
+ builder.SetInsertPoint(
+ llvm::BasicBlock::Create(context, "entry", function));
+ builder.CreateCall(callback);
+ builder.CreateRetVoid();
+ if (isCtor)
+ llvm::appendToGlobalCtors(*module, function, 0);
+ else
+ llvm::appendToGlobalDtors(*module, function, 0);
+ };
+ addCtorOrDtor("ctor_callback", "ctor", true);
+ addCtorOrDtor("dtor_callback", "dtor", false);
+
+ auto *farData = new llvm::GlobalVariable(*module, builder.getInt32Ty(), false,
+ llvm::GlobalValue::ExternalLinkage,
+ nullptr, "far_data");
+ auto *addressFnTy = llvm::FunctionType::get(builder.getInt64Ty(), false);
+ auto *addressFn =
+ llvm::Function::Create(addressFnTy, llvm::GlobalValue::ExternalLinkage,
+ "get_far_data_address", *module);
+ builder.SetInsertPoint(llvm::BasicBlock::Create(context, "entry", addressFn));
+ builder.CreateRet(builder.CreatePtrToInt(farData, builder.getInt64Ty()));
+ return module;
+}
+
+TEST(MLIRExecutionEngine, ConstructorsAndFarExternalAddressOnWindowsArm64) {
+ CtorCount = 0;
+ DtorCount = 0;
+
+ MLIRContext context;
+ OwningOpRef<ModuleOp> module = ModuleOp::create(UnknownLoc::get(&context));
+ FarMemoryMapper mapper(reinterpret_cast<uintptr_t>(&FarData));
+ ExecutionEngineOptions options;
+ options.llvmModuleBuilder = buildFarAddressModule;
+ options.sectionMemoryMapper = &mapper;
+
+ auto jitOrError = ExecutionEngine::create(*module, options);
+ if (!jitOrError)
+ FAIL() << llvm::toString(jitOrError.takeError());
+ std::unique_ptr<ExecutionEngine> jit = std::move(*jitOrError);
+ jit->registerSymbols([](llvm::orc::MangleAndInterner interner) {
+ llvm::orc::SymbolMap symbols;
+ symbols[interner("far_data")] = {llvm::orc::ExecutorAddr::fromPtr(&FarData),
+ llvm::JITSymbolFlags::Exported};
+ symbols[interner("ctor_callback")] = {
+ llvm::orc::ExecutorAddr::fromPtr(&ctorCallback),
+ llvm::JITSymbolFlags::Exported};
+ symbols[interner("dtor_callback")] = {
+ llvm::orc::ExecutorAddr::fromPtr(&dtorCallback),
+ llvm::JITSymbolFlags::Exported};
+ return symbols;
+ });
+
+ EXPECT_EQ(CtorCount, 0);
+ jit->initialize();
+ EXPECT_EQ(CtorCount, 1);
+
+ auto addressOrError = jit->lookup("get_far_data_address");
+ if (!addressOrError)
+ FAIL() << llvm::toString(addressOrError.takeError());
+ auto getAddress = reinterpret_cast<uintptr_t (*)()>(*addressOrError);
+ EXPECT_EQ(getAddress(), reinterpret_cast<uintptr_t>(&FarData));
+ EXPECT_GT(mapper.getMinimumDistance(), FourGiB);
+
+ jit.reset();
+ EXPECT_EQ(DtorCount, 1);
+}
+} // namespace
+
+#endif // Windows/AArch64
More information about the Mlir-commits
mailing list