[llvm] [AArch64][BOLT] Fold local-exec TLS relocations into loads and stores (PR #215531)
Jiang Ning via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 23:28:44 PDT 2026
https://github.com/jiangning26 updated https://github.com/llvm/llvm-project/pull/215531
>From 222c32185d3679ba4775af8bd6a8839aaa5036ff Mon Sep 17 00:00:00 2001
From: JiangNing <jiangninghx at foxmail.com>
Date: Tue, 11 Aug 2026 17:40:12 +0800
Subject: [PATCH] [AArch64][BOLT] Fold local-exec TLS relocations into loads
and stores
Emit the low part of the 12- and 24-bit local-exec sequences as ADDlow
so scalar loads and stores can fold it into their addressing mode.
This re-lands the local-exec part of r327316, reverted in r327503 when
linkers lacked the required relocations. Keep 128-bit accesses unfolded
because GNU bfd still lacks LDST128 support.
Teach BOLT to classify the folded TLS relocations in binaries linked
with --emit-relocs.
---
bolt/lib/Core/Relocation.cpp | 8 +++
bolt/test/AArch64/tls.c | 5 ++
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 19 ++++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 13 ++---
.../CodeGen/AArch64/arm64-tls-local-exec.ll | 53 ++++++++++++++++---
llvm/test/CodeGen/AArch64/win-tls.ll | 10 ++++
6 files changed, 93 insertions(+), 15 deletions(-)
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index b0f6b6ce0eddc..55d5e07042897 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -606,6 +606,14 @@ static bool isTLSAArch64(uint32_t Type) {
case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:
case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
+ case ELF::R_AARCH64_TLSLE_LDST8_TPREL_LO12:
+ case ELF::R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
+ case ELF::R_AARCH64_TLSLE_LDST16_TPREL_LO12:
+ case ELF::R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
+ case ELF::R_AARCH64_TLSLE_LDST32_TPREL_LO12:
+ case ELF::R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
+ case ELF::R_AARCH64_TLSLE_LDST64_TPREL_LO12:
+ case ELF::R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0:
case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
case ELF::R_AARCH64_TLSDESC_LD64_LO12:
diff --git a/bolt/test/AArch64/tls.c b/bolt/test/AArch64/tls.c
index 947a5f587f293..5d19aa07fea8e 100644
--- a/bolt/test/AArch64/tls.c
+++ b/bolt/test/AArch64/tls.c
@@ -5,6 +5,8 @@ __thread struct str {
int b;
} tbssstruct = {}, tdatastruct = {4, 2};
+__thread int directaccess;
+
extern __thread struct str extstruct;
extern void processAddr(volatile void *);
@@ -18,6 +20,9 @@ int main() {
processAddr(&tbssstruct.b);
processAddr(&tdatastruct.b);
+ // R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC for a direct access
+ directaccess++;
+
// The R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 and
// R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC relocations
processAddr(&extstruct.b);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index f22806678211f..ddbe7f24dc39a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -1254,6 +1254,18 @@ static bool isWorthFoldingADDlow(SDValue N) {
return true;
}
+/// Check whether \p GAN is the low part of a TLS address computation, i.e. the
+/// second operand of an ADDlow. The target flags on their own do not tell the
+/// ELF local-exec (:tprel_lo12: and :tprel_lo12_nc:) cases apart from other
+/// uses, so callers that depend on local-exec semantics have to check the
+/// object format as well. Local dynamic never gets here because it does not
+/// build an ADDlow.
+static bool isTLSLo12(const GlobalAddressSDNode *GAN) {
+ unsigned Flags = GAN->getTargetFlags();
+ return (Flags & (AArch64II::MO_TLS | AArch64II::MO_FRAGMENT)) ==
+ (AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
+}
+
/// Check if the immediate offset is valid as a scaled immediate.
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
unsigned Size) {
@@ -1349,8 +1361,13 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
if (!GAN)
return true;
+ // Folding the low part of an ELF local-exec TLS address into a 128-bit
+ // access needs R_AARCH64_TLSLE_LDST128_TPREL_LO12 or its NC variant, which
+ // the GNU bfd linker does not support, so keep materialising the address
+ // with an add.
if (GAN->getOffset() % Size == 0 &&
- GAN->getGlobal()->getPointerAlignment(DL) >= Size)
+ GAN->getGlobal()->getPointerAlignment(DL) >= Size &&
+ !(Size > 8 && Subtarget->isTargetELF() && isTLSLo12(GAN)))
return true;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6003db1c72449..75cc5b25bf0dc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11464,10 +11464,7 @@ SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
// add x0, x0, :tprel_lo12:a
SDValue Var = DAG.getTargetGlobalAddress(
GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
- return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
- Var,
- DAG.getTargetConstant(0, DL, MVT::i32)),
- 0);
+ return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ThreadBase, Var);
}
case 24: {
@@ -11483,10 +11480,10 @@ SDValue AArch64TargetLowering::LowerELFTLSLocalExec(const GlobalValue *GV,
HiVar,
DAG.getTargetConstant(0, DL, MVT::i32)),
0);
- return SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, Addr,
- LoVar,
- DAG.getTargetConstant(0, DL, MVT::i32)),
- 0);
+ // Emit the low part as an ADDlow so that it can be folded into the
+ // addressing mode of a following load or store, turning the add into a
+ // :tprel_lo12_nc: relocation on the memory access itself.
+ return DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, LoVar);
}
case 32: {
diff --git a/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll b/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
index 59d5500ce534e..a30a441abfe1f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-tls-local-exec.ll
@@ -27,24 +27,24 @@
; RUN: llc -mtriple=arm64-none-linux-gnu -filetype=obj < %s -code-model=large | llvm-objdump -r - | FileCheck --check-prefix=CHECK-24-RELOC %s
@local_exec_var = thread_local(localexec) global i32 0
+ at local_exec_var64 = thread_local(localexec) global i64 0
+ at vec_local_exec_var = thread_local(localexec) global <2 x i64> zeroinitializer, align 16
define i32 @test_local_exec() {
; CHECK-LABEL: test_local_exec:
%val = load i32, ptr @local_exec_var
; CHECK-12: mrs x[[R1:[0-9]+]], TPIDR_EL0
-; CHECK-12: add x[[R2:[0-9]+]], x[[R1]], :tprel_lo12:local_exec_var
-; CHECK-12: ldr w0, [x[[R2]]]
+; CHECK-12: ldr w0, [x[[R1]], :tprel_lo12:local_exec_var]
-; CHECK-12-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12
+; CHECK-12-RELOC: R_AARCH64_TLSLE_LDST32_TPREL_LO12
; CHECK-24: mrs x[[R1:[0-9]+]], TPIDR_EL0
; CHECK-24: add x[[R2:[0-9]+]], x[[R1]], :tprel_hi12:local_exec_var
-; CHECK-24: add x[[R3:[0-9]+]], x[[R2]], :tprel_lo12_nc:local_exec_var
-; CHECK-24: ldr w0, [x[[R3]]]
+; CHECK-24: ldr w0, [x[[R2]], :tprel_lo12_nc:local_exec_var]
; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_HI12
-; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12_NC
+; CHECK-24-RELOC: R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC
; CHECK-32: movz x[[R2:[0-9]+]], #:tprel_g1:local_exec_var
; CHECK-32: mrs x[[R1:[0-9]+]], TPIDR_EL0
@@ -66,6 +66,24 @@ define i32 @test_local_exec() {
ret i32 %val
}
+define void @test_local_exec_store64(i64 %val) {
+; CHECK-LABEL: test_local_exec_store64:
+ store i64 %val, ptr @local_exec_var64
+
+; CHECK-12: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-12: str x0, [x[[R1]], :tprel_lo12:local_exec_var64]
+
+; CHECK-12-RELOC: R_AARCH64_TLSLE_LDST64_TPREL_LO12
+
+; CHECK-24: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-24: add x[[R2:[0-9]+]], x[[R1]], :tprel_hi12:local_exec_var64
+; CHECK-24: str x0, [x[[R2]], :tprel_lo12_nc:local_exec_var64]
+
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_HI12 local_exec_var64
+; CHECK-24-RELOC-NEXT: R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC local_exec_var64
+ ret void
+}
+
define ptr @test_local_exec_addr() {
; CHECK-LABEL: test_local_exec_addr:
ret ptr @local_exec_var
@@ -104,3 +122,26 @@ define ptr @test_local_exec_addr() {
; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G1_NC
; CHECK-48-RELOC: R_AARCH64_TLSLE_MOVW_TPREL_G0_NC
}
+
+; A 128-bit access would need R_AARCH64_TLSLE_LDST128_TPREL_LO12 or its NC
+; variant, which not every linker implements, so the low part stays in a
+; separate add.
+define <2 x i64> @test_local_exec_128bit() {
+; CHECK-LABEL: test_local_exec_128bit:
+ %val = load <2 x i64>, ptr @vec_local_exec_var
+
+; CHECK-12: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-12: add x[[R2:[0-9]+]], x[[R1]], :tprel_lo12:vec_local_exec_var
+; CHECK-12: ldr q0, [x[[R2]]]
+
+; CHECK-12-RELOC: R_AARCH64_TLSLE_ADD_TPREL_LO12 vec_local_exec_var
+
+; CHECK-24: mrs x[[R1:[0-9]+]], TPIDR_EL0
+; CHECK-24: add x[[R2:[0-9]+]], x[[R1]], :tprel_hi12:vec_local_exec_var
+; CHECK-24: add x[[R3:[0-9]+]], x[[R2]], :tprel_lo12_nc:vec_local_exec_var
+; CHECK-24: ldr q0, [x[[R3]]]
+
+; CHECK-24-RELOC: R_AARCH64_TLSLE_ADD_TPREL_HI12 vec_local_exec_var
+; CHECK-24-RELOC-NEXT: R_AARCH64_TLSLE_ADD_TPREL_LO12_NC vec_local_exec_var
+ ret <2 x i64> %val
+}
diff --git a/llvm/test/CodeGen/AArch64/win-tls.ll b/llvm/test/CodeGen/AArch64/win-tls.ll
index e448282b7c4cf..f184ad74a9e25 100644
--- a/llvm/test/CodeGen/AArch64/win-tls.ll
+++ b/llvm/test/CodeGen/AArch64/win-tls.ll
@@ -3,6 +3,7 @@
@tlsVar = thread_local global i32 0
@tlsVar8 = thread_local global i8 0
@tlsVar64 = thread_local global i64 0
+ at tlsVar128 = thread_local global <2 x i64> zeroinitializer
define i32 @getVar() {
%1 = load i32, ptr @tlsVar
@@ -28,6 +29,11 @@ define i64 @getVar64() {
ret i64 %1
}
+define <2 x i64> @getVar128() {
+ %1 = load <2 x i64>, ptr @tlsVar128
+ ret <2 x i64> %1
+}
+
; CHECK-LABEL: getVar
; CHECK: adrp [[TLS_INDEX_ADDR:x[0-9]+]], _tls_index
; CHECK: ldr [[TLS_POINTER:x[0-9]+]], [x18, #88]
@@ -62,3 +68,7 @@ define i64 @getVar64() {
; CHECK-LABEL: getVar64
; CHECK: add [[TLS:x[0-9]+]], [[TLS]], :secrel_hi12:tlsVar64
; CHECK: ldr x0, [[[TLS]], :secrel_lo12:tlsVar64]
+
+; CHECK-LABEL: getVar128
+; CHECK: add [[TLS:x[0-9]+]], [[TLS]], :secrel_hi12:tlsVar128
+; CHECK: ldr q0, [[[TLS]], :secrel_lo12:tlsVar128]
More information about the llvm-commits
mailing list