[llvm-branch-commits] [llvm] [AArch64] ldiapp for 128-bit SC load with rcpc3 (PR #208443)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jul 9 05:13:16 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Tomas Matheson (tommat01)
<details>
<summary>Changes</summary>
---
Full diff: https://github.com/llvm/llvm-project/pull/208443.diff
5 Files Affected:
- (modified) llvm/lib/CodeGen/AtomicExpandPass.cpp (+10-13)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+27-3)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+4)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll (+8-16)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll (+8-16)
``````````diff
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 6985a7a48147c..2d86592aac8d6 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -94,8 +94,8 @@ class AtomicExpandImpl {
bool bracketInstWithFences(Instruction *I, AtomicOrdering Order);
bool tryInsertTrailingSeqCstFence(Instruction *AtomicI);
template <typename AtomicInst>
- bool tryInsertFencesForAtomic(AtomicInst *AtomicI, bool OrderingRequiresFence,
- AtomicOrdering NewOrdering);
+ bool tryInsertFencesForAtomic(AtomicInst *AtomicI,
+ bool OrderingRequiresFence);
IntegerType *getCorrespondingIntegerType(Type *T, const DataLayout &DL);
LoadInst *convertAtomicLoadToIntegerType(LoadInst *LI);
bool tryExpandAtomicLoad(LoadInst *LI);
@@ -321,12 +321,11 @@ bool AtomicExpandImpl::tryInsertTrailingSeqCstFence(Instruction *AtomicI) {
template <typename AtomicInst>
bool AtomicExpandImpl::tryInsertFencesForAtomic(AtomicInst *AtomicI,
- bool OrderingRequiresFence,
- AtomicOrdering NewOrdering) {
+ bool OrderingRequiresFence) {
bool ShouldInsertFences = TLI->shouldInsertFencesForAtomic(AtomicI);
if (OrderingRequiresFence && ShouldInsertFences) {
AtomicOrdering FenceOrdering = AtomicI->getOrdering();
- AtomicI->setOrdering(NewOrdering);
+ AtomicI->setOrdering(TLI->atomicOperationOrderAfterFenceSplit(AtomicI));
return bracketInstWithFences(AtomicI, FenceOrdering);
}
if (!ShouldInsertFences)
@@ -351,8 +350,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
MadeChange = true;
}
- MadeChange |= tryInsertFencesForAtomic(
- LI, isAcquireOrStronger(LI->getOrdering()), AtomicOrdering::Monotonic);
+ MadeChange |=
+ tryInsertFencesForAtomic(LI, isAcquireOrStronger(LI->getOrdering()));
MadeChange |= tryExpandAtomicLoad(LI);
return MadeChange;
@@ -374,8 +373,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
MadeChange = true;
}
- MadeChange |= tryInsertFencesForAtomic(
- SI, isReleaseOrStronger(SI->getOrdering()), AtomicOrdering::Monotonic);
+ MadeChange |=
+ tryInsertFencesForAtomic(SI, isReleaseOrStronger(SI->getOrdering()));
MadeChange |= tryExpandAtomicStore(SI);
return MadeChange;
@@ -395,10 +394,8 @@ bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
}
MadeChange |= tryInsertFencesForAtomic(
- RMWI,
- isReleaseOrStronger(RMWI->getOrdering()) ||
- isAcquireOrStronger(RMWI->getOrdering()),
- TLI->atomicOperationOrderAfterFenceSplit(RMWI));
+ RMWI, isReleaseOrStronger(RMWI->getOrdering()) ||
+ isAcquireOrStronger(RMWI->getOrdering()));
// There are two different ways of expanding RMW instructions:
// - into a load if it is idempotent
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 642afb8922af3..b7ab4d99bf87b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -32024,7 +32024,7 @@ bool AArch64TargetLowering::isOpSuitableForRCPC3(const Instruction *I) const {
if (auto LI = dyn_cast<LoadInst>(I))
return LI->getType()->getPrimitiveSizeInBits() == 128 &&
LI->getAlign() >= Align(16) &&
- LI->getOrdering() == AtomicOrdering::Acquire;
+ isAcquireOrStronger(LI->getOrdering());
if (auto SI = dyn_cast<StoreInst>(I))
return SI->getValueOperand()->getType()->getPrimitiveSizeInBits() == 128 &&
@@ -32036,6 +32036,10 @@ bool AArch64TargetLowering::isOpSuitableForRCPC3(const Instruction *I) const {
bool AArch64TargetLowering::shouldInsertFencesForAtomic(
const Instruction *I) const {
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ LI && LI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForRCPC3(LI))
+ return true; // need leading LDAR
if (isOpSuitableForRCPC3(I))
return false;
if (isOpSuitableForLSE128(I))
@@ -32074,11 +32078,21 @@ bool AArch64TargetLowering::shouldInsertTrailingSeqCstFenceForAtomicStore(
return !Subtarget->hasLSE();
}
+AtomicOrdering AArch64TargetLowering::atomicOperationOrderAfterFenceSplit(
+ const Instruction *I) const {
+ if (auto *LI = dyn_cast<LoadInst>(I);
+ LI && LI->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForRCPC3(LI))
+ return AtomicOrdering::Acquire;
+
+ return TargetLoweringBase::atomicOperationOrderAfterFenceSplit(I);
+}
+
Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder,
Instruction *Inst,
AtomicOrdering Ord) const {
- // Keep seq_cst 128-bit LSE2 loads ordered against v8.0-style seq_cst LL/SC
- // stores by emitting an (unused) LDAR before the LDP.
+ // Keep seq_cst 128-bit LSE2/RCPC3 loads ordered against v8.0-style seq_cst
+ // LL/SC stores by emitting an (unused) LDAR before the 128-bit load.
if (auto *LI = dyn_cast<LoadInst>(Inst);
LI && Ord == AtomicOrdering::SequentiallyConsistent &&
isOpSuitableForLDPSTP(LI)) {
@@ -32092,6 +32106,16 @@ Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder,
return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
}
+Instruction *AArch64TargetLowering::emitTrailingFence(
+ IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const {
+ if (auto *LI = dyn_cast<LoadInst>(Inst);
+ LI && Ord == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForRCPC3(LI))
+ return nullptr;
+
+ return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
+}
+
// Loads and stores less than 128-bits are already atomic; ones above that
// are doomed anyway, so defer to the default libcall and blame the OS when
// things go wrong.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 69013d4010122..cc8cec8a83738 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -348,6 +348,10 @@ class AArch64TargetLowering : public TargetLowering {
Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
AtomicOrdering Ord) const override;
+ Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
+ AtomicOrdering
+ atomicOperationOrderAfterFenceSplit(const Instruction *I) const override;
Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
AtomicOrdering Ord) const override;
Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
index 04f7572cf4332..4618e53a54f7a 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
@@ -272,13 +272,11 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; -O0-LABEL: load_atomic_i128_aligned_seq_cst:
; -O0: ldar x8, [x0]
-; -O0: ldp x0, x1, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x0, x1, [x0]
;
; -O1-LABEL: load_atomic_i128_aligned_seq_cst:
; -O1: ldar xzr, [x0]
-; -O1: ldp x0, x1, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
@@ -286,13 +284,11 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; -O0-LABEL: load_atomic_i128_aligned_seq_cst_const:
; -O0: ldar x8, [x0]
-; -O0: ldp x0, x1, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x0, x1, [x0]
;
; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const:
; -O1: ldar xzr, [x0]
-; -O1: ldp x0, x1, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
@@ -580,13 +576,11 @@ define dso_local i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %pt
define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) {
; -O0-LABEL: load_atomic_fp128_aligned_seq_cst:
; -O0: ldar x8, [x0]
-; -O0: ldp x9, x8, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x9, x8, [x0]
;
; -O1-LABEL: load_atomic_fp128_aligned_seq_cst:
; -O1: ldar xzr, [x0]
-; -O1: ldp x8, x9, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x8, x9, [x0]
%r = load atomic fp128, ptr %ptr seq_cst, align 16
ret fp128 %r
}
@@ -594,13 +588,11 @@ define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) {
define dso_local fp128 @load_atomic_fp128_aligned_seq_cst_const(ptr readonly %ptr) {
; -O0-LABEL: load_atomic_fp128_aligned_seq_cst_const:
; -O0: ldar x8, [x0]
-; -O0: ldp x9, x8, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x9, x8, [x0]
;
; -O1-LABEL: load_atomic_fp128_aligned_seq_cst_const:
; -O1: ldar xzr, [x0]
-; -O1: ldp x8, x9, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x8, x9, [x0]
%r = load atomic fp128, ptr %ptr seq_cst, align 16
ret fp128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
index b25af543ff8fd..222a1a92741f5 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
@@ -272,13 +272,11 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; -O0-LABEL: load_atomic_i128_aligned_seq_cst:
; -O0: ldar x8, [x0]
-; -O0: ldp x0, x1, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x0, x1, [x0]
;
; -O1-LABEL: load_atomic_i128_aligned_seq_cst:
; -O1: ldar xzr, [x0]
-; -O1: ldp x0, x1, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
@@ -286,13 +284,11 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; -O0-LABEL: load_atomic_i128_aligned_seq_cst_const:
; -O0: ldar x8, [x0]
-; -O0: ldp x0, x1, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x0, x1, [x0]
;
; -O1-LABEL: load_atomic_i128_aligned_seq_cst_const:
; -O1: ldar xzr, [x0]
-; -O1: ldp x0, x1, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
@@ -580,13 +576,11 @@ define dso_local i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %pt
define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) {
; -O0-LABEL: load_atomic_fp128_aligned_seq_cst:
; -O0: ldar x8, [x0]
-; -O0: ldp x8, x9, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x8, x9, [x0]
;
; -O1-LABEL: load_atomic_fp128_aligned_seq_cst:
; -O1: ldar xzr, [x0]
-; -O1: ldp x8, x9, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x8, x9, [x0]
%r = load atomic fp128, ptr %ptr seq_cst, align 16
ret fp128 %r
}
@@ -594,13 +588,11 @@ define dso_local fp128 @load_atomic_fp128_aligned_seq_cst(ptr %ptr) {
define dso_local fp128 @load_atomic_fp128_aligned_seq_cst_const(ptr readonly %ptr) {
; -O0-LABEL: load_atomic_fp128_aligned_seq_cst_const:
; -O0: ldar x8, [x0]
-; -O0: ldp x8, x9, [x0]
-; -O0: dmb ish
+; -O0: ldiapp x8, x9, [x0]
;
; -O1-LABEL: load_atomic_fp128_aligned_seq_cst_const:
; -O1: ldar xzr, [x0]
-; -O1: ldp x8, x9, [x0]
-; -O1: dmb ish
+; -O1: ldiapp x8, x9, [x0]
%r = load atomic fp128, ptr %ptr seq_cst, align 16
ret fp128 %r
}
``````````
</details>
https://github.com/llvm/llvm-project/pull/208443
More information about the llvm-branch-commits
mailing list