[llvm] [AArch64] fix 128-bit Sequentially Consistent load (PR #206936)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 03:13:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-globalisel
Author: Tomas Matheson (tommat01)
<details>
<summary>Changes</summary>
Fixes https://github.com/llvm/llvm-project/issues/81978
---
Patch is 33.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/206936.diff
11 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+21)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+4)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll (+2-2)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll (+209-87)
- (modified) llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll (+187-69)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..339fb66db23f7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -31707,6 +31707,27 @@ bool AArch64TargetLowering::shouldInsertTrailingSeqCstFenceForAtomicStore(
return !Subtarget->hasLSE();
}
+Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder,
+ Instruction *Inst,
+ AtomicOrdering Ord) const {
+ // Keep seq_cst 128-bit LSE2 loads ordered against v8.0-style seq_cst LL/SC
+ // stores by placing the full barrier before the single-copy atomic load.
+ if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForLDPSTP(Inst))
+ return Builder.CreateFence(Ord);
+
+ return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
+}
+
+Instruction *AArch64TargetLowering::emitTrailingFence(
+ IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const {
+ if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForLDPSTP(Inst))
+ return nullptr;
+
+ return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
+}
+
// Loads and stores less than 128-bits are already atomic; ones above that
// are doomed anyway, so defer to the default libcall and blame the OS when
// things go wrong.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index ad67705377661..9f1078b0a33ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -346,6 +346,10 @@ class AArch64TargetLowering : public TargetLowering {
bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context,
EVT VT) const override;
+ Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
+ Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
AtomicOrdering Ord) const override;
Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
index 0e9c29758244a..9fce4b85c7d8a 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
index 2f030b0e23ab1..53340805000de 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
index 435f0a0c01765..e8ad2e7664f52 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
@@ -271,16 +271,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index f91ab9fa3e8ca..0d55db9c69a3b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -383,8 +383,8 @@ define i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) {
define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0, #64]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0, #64]
%gep = getelementptr inbounds i128, ptr %ptr, i32 4
%r = load atomic i128, ptr %gep seq_cst, align 16
ret i128 %r
@@ -392,8 +392,8 @@ define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0, #64]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0, #64]
%gep = getelementptr inbounds i128, ptr %ptr, i32 4
%r = load atomic i128, ptr %gep seq_cst, align 16
ret i128 %r
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
index 08f7dd32c1280..e7082207f1e78 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
index 74e612d59858f..230674e567c15 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
index de83b702d988a..9d94c60d67eb7 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
@@ -271,16 +271,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
+; CHECK: ldp x0, x1, [x0]
%r = load atomic i128, ptr %ptr seq_cst, align 16
ret i128 %r
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
index fe7e24c2d8ba5..00195e8e6c2e5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
@@ -1,60 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+v8.4a %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse2 %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
-define void @test_atomic_load(ptr %addr) {
-; CHECK-LABEL: test_atomic_load:
-
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_atomic_load_monotonic(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_monotonic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%res.0 = load atomic i128, ptr %addr monotonic, align 16
store i128 %res.0, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_unordered(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_unordered:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%res.1 = load atomic i128, ptr %addr unordered, align 16
store i128 %res.1, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: dmb ish
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_acquire(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_acquire:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: dmb ishld
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%res.2 = load atomic i128, ptr %addr acquire, align 16
store i128 %res.2, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: dmb ish
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_seq_cst(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_seq_cst:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%res.3 = load atomic i128, ptr %addr seq_cst, align 16
store i128 %res.3, ptr %addr
+ ret void
+}
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #8]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_atomic_load_offset_8(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0, #8]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.1 = getelementptr i8, ptr %addr, i32 8
%res.5 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.5, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #504]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_offset_504(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_504:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0, #504]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.2 = getelementptr i8, ptr %addr, i32 504
%res.6 = load atomic i128, ptr %addr8.2 monotonic, align 16
store i128 %res.6, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #-512]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_offset_negative_512(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_negative_512:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ldp x8, x9, [x0, #-512]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.3 = getelementptr i8, ptr %addr, i32 -512
%res.7 = load atomic i128, ptr %addr8.3 monotonic, align 16
store i128 %res.7, ptr %addr
@@ -64,21 +107,38 @@ define void @test_atomic_load(ptr %addr) {
define void @test_libcall_load(ptr %addr) {
; CHECK-LABEL: test_libcall_load:
-; CHECK: bl __atomic_load
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: .cfi_offset w19, -8
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: mov x19, x0
+; CHECK-NEXT: mov x2, sp
+; CHECK-NEXT: mov w0, #16 // =0x10
+; CHECK-NEXT: mov x1, x19
+; CHECK-NEXT: mov w3, wzr
+; CHECK-NEXT: bl __atomic_load
+; CHECK-NEXT: ldr q0, [sp]
+; CHECK-NEXT: str q0, [x19]
+; CHECK-NEXT: ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT: add sp, sp, #32
+; CHECK-NEXT: ret
%res.8 = load atomic i128, ptr %addr unordered, align 8
store i128 %res.8, ptr %addr
ret void
}
-define void @test_nonfolded_load1(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load1:
-
-; CHECK: add x[[ADDR:[0-9]+]], x0, #4
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_4(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #4
+; CHECK-NEXT: ldp x8, x9, [x8]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.1 = getelementptr i8, ptr %addr, i32 4
%res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.1, ptr %addr
@@ -86,14 +146,15 @@ define void @test_nonfolded_load1(ptr %addr) {
ret void
}
-define void @test_nonfolded_load2(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load2:
-
-; CHECK: add x[[ADDR:[0-9]+]], x0, #512
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_512(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_512:
+; CHECK: // %bb.0:
+; CHECK-NEXT: add x8, x0, #512
+; CHECK-NEXT: ldp x8, x9, [x8]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.1 = getelementptr i8, ptr %addr, i32 512
%res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.1, ptr %addr
@@ -101,14 +162,15 @@ define void @test_nonfolded_load2(ptr %addr) {
ret void
}
-define void @test_nonfolded_load3(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load3:
-
-; CHECK: sub x[[ADDR:[0-9]+]], x0, #520
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_negative_520(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_negative_520:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #520
+; CHECK-NEXT: ldp x8, x9, [x8]
+; CHECK-NEXT: mov v0.d[0], x8
+; CHECK-NEXT: mov v0.d[1], x9
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: ret
%addr8.1 = getelementptr i8, ptr %addr, i32 -520
%res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.1, ptr %addr
@@ -116,34 +178,76 @@ define void @test_nonfolded_load3(ptr %addr) {
ret void
}
-define void @test_atomic_store(ptr %addr, i128 %val) {
-; CHECK-LABEL: test_atomic_store:
-
-; CHECK: stp x2, x3, [x0]
+define void @test_atomic_store_monotonic(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_monotonic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x2, x3, [x0]
+; CHECK-NEXT: ret
store atomic i128 %val, ptr %addr monotonic, align 16
-; CHECK: stp x2, x3, [x0]
+ ret void
+}
+
+define void @test_atomic_store_unordered(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_unordered:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x2, x3, [x0]
+; CHECK-NEXT: ret
store atomic i128 %val, ptr %addr unordered, align 16
-; CHECK: dmb ish
-; CHECK: stp x2, x3, [x0]
+ ret void
+}
+
+define void @test_atomic_store_release(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_release:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: stp x2, x3, [x0]
+; CHECK-NEXT: ret
store atomic i128 %val, ptr %addr release, align 16
-; CHECK: dmb ish
-; CHECK: stp x2, x3, [x0]
-; CHECK: dmb ish
+ ret void
+}
+
+define void @test_atomic_store_seq_cst(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_seq_cst:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: stp x2, x3, [x0]
+; CHECK-NEXT: dmb ish
+; CHECK-NEXT: ret
store atomic i128 %val, ptr %addr seq_cst, align 16
+ ret void
+}
-; CHECK: stp x2, x3, [x0, #8]
+define void @test_atomic_store_offset_8(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x2, x3, [x0, #8]
+; CHECK-NEXT: ret
%addr8.1 = getelementptr i8, ptr %addr, i32 8
store atomic i128 %val, ptr %addr8.1 monotonic, align 16
-; CHECK: stp x2, x3, [x0, #504]
+ ret void
+}
+
+define void @test_atomic_store_offset_504(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_504:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x2, x3, [x0, #504]
+; CHECK-NEXT: ret
%addr8.2 = getelementptr i8, ptr %addr, i32 504
store atomic i128 %val, ptr %addr8.2 monotonic, align 16
-; CHECK: stp x2, x3, [x0, #-512]
+ ret void
+}
+
+define void @test_atomic_store_offset_negative_512(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_negative_512:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stp x2, x3, [x0, #-512]
+; CHECK-NEXT: ret
%addr8.3 = getelementptr i8, ptr %addr, i32 -512
store atomic i...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/206936
More information about the llvm-commits
mailing list