[llvm] [AArch64] fix 128-bit Sequentially Consistent load (PR #206936)

via llvm-commits llvm-commits at lists.llvm.org
Wed Jul 1 03:13:56 PDT 2026


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-llvm-globalisel

Author: Tomas Matheson (tommat01)

<details>
<summary>Changes</summary>

Fixes https://github.com/llvm/llvm-project/issues/81978

---

Patch is 33.97 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/206936.diff


11 Files Affected:

- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+21) 
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+4) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll (+2-2) 
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll (+209-87) 
- (modified) llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll (+187-69) 


``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..339fb66db23f7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -31707,6 +31707,27 @@ bool AArch64TargetLowering::shouldInsertTrailingSeqCstFenceForAtomicStore(
   return !Subtarget->hasLSE();
 }
 
+Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder,
+                                                     Instruction *Inst,
+                                                     AtomicOrdering Ord) const {
+  // Keep seq_cst 128-bit LSE2 loads ordered against v8.0-style seq_cst LL/SC
+  // stores by placing the full barrier before the single-copy atomic load.
+  if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent &&
+      isOpSuitableForLDPSTP(Inst))
+    return Builder.CreateFence(Ord);
+
+  return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
+}
+
+Instruction *AArch64TargetLowering::emitTrailingFence(
+    IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const {
+  if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent &&
+      isOpSuitableForLDPSTP(Inst))
+    return nullptr;
+
+  return TargetLoweringBase::emitTrailingFence(Builder, Inst, Ord);
+}
+
 // Loads and stores less than 128-bits are already atomic; ones above that
 // are doomed anyway, so defer to the default libcall and blame the OS when
 // things go wrong.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index ad67705377661..9f1078b0a33ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -346,6 +346,10 @@ class AArch64TargetLowering : public TargetLowering {
   bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context,
                                                  EVT VT) const override;
 
+  Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
+                                AtomicOrdering Ord) const override;
+  Instruction *emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst,
+                                 AtomicOrdering Ord) const override;
   Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
                         AtomicOrdering Ord) const override;
   Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
index 0e9c29758244a..9fce4b85c7d8a 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
index 2f030b0e23ab1..53340805000de 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
index 435f0a0c01765..e8ad2e7664f52 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
@@ -271,16 +271,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index f91ab9fa3e8ca..0d55db9c69a3b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -383,8 +383,8 @@ define i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) {
 
 define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0, #64]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0, #64]
     %gep = getelementptr inbounds i128, ptr %ptr, i32 4
     %r = load atomic i128, ptr %gep seq_cst, align 16
     ret i128 %r
@@ -392,8 +392,8 @@ define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 
 define i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0, #64]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0, #64]
     %gep = getelementptr inbounds i128, ptr %ptr, i32 4
     %r = load atomic i128, ptr %gep seq_cst, align 16
     ret i128 %r
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
index 08f7dd32c1280..e7082207f1e78 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
index 74e612d59858f..230674e567c15 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
@@ -273,16 +273,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
index de83b702d988a..9d94c60d67eb7 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
@@ -271,16 +271,16 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
 
 define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
 ; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
-; CHECK:    ldp x0, x1, [x0]
 ; CHECK:    dmb ish
+; CHECK:    ldp x0, x1, [x0]
     %r = load atomic i128, ptr %ptr seq_cst, align 16
     ret i128 %r
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
index fe7e24c2d8ba5..00195e8e6c2e5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
@@ -1,60 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+v8.4a %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse2 %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
 
-define void @test_atomic_load(ptr %addr) {
-; CHECK-LABEL: test_atomic_load:
-
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_atomic_load_monotonic(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_monotonic:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %res.0 = load atomic i128, ptr %addr monotonic, align 16
   store i128 %res.0, ptr %addr
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+  ret void
+}
+
+define void @test_atomic_load_unordered(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_unordered:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %res.1 = load atomic i128, ptr %addr unordered, align 16
   store i128 %res.1, ptr %addr
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: dmb ish
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+  ret void
+}
+
+define void @test_atomic_load_acquire(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_acquire:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0]
+; CHECK-NEXT:    dmb ishld
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %res.2 = load atomic i128, ptr %addr acquire, align 16
   store i128 %res.2, ptr %addr
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: dmb ish
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+  ret void
+}
+
+define void @test_atomic_load_seq_cst(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_seq_cst:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dmb ish
+; CHECK-NEXT:    ldp x8, x9, [x0]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %res.3 = load atomic i128, ptr %addr seq_cst, align 16
   store i128 %res.3, ptr %addr
 
+  ret void
+}
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #8]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_atomic_load_offset_8(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0, #8]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.1 = getelementptr i8,  ptr %addr, i32 8
   %res.5 = load atomic i128, ptr %addr8.1 monotonic, align 16
   store i128 %res.5, ptr %addr
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #504]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+  ret void
+}
+
+define void @test_atomic_load_offset_504(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_504:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0, #504]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.2 = getelementptr i8,  ptr %addr, i32 504
   %res.6 = load atomic i128, ptr %addr8.2 monotonic, align 16
   store i128 %res.6, ptr %addr
 
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #-512]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+  ret void
+}
+
+define void @test_atomic_load_offset_negative_512(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_offset_negative_512:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp x8, x9, [x0, #-512]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.3 = getelementptr i8,  ptr %addr, i32 -512
   %res.7 = load atomic i128, ptr %addr8.3 monotonic, align 16
   store i128 %res.7, ptr %addr
@@ -64,21 +107,38 @@ define void @test_atomic_load(ptr %addr) {
 
 define void @test_libcall_load(ptr %addr) {
 ; CHECK-LABEL: test_libcall_load:
-; CHECK: bl __atomic_load
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    mov x2, sp
+; CHECK-NEXT:    mov w0, #16 // =0x10
+; CHECK-NEXT:    mov x1, x19
+; CHECK-NEXT:    mov w3, wzr
+; CHECK-NEXT:    bl __atomic_load
+; CHECK-NEXT:    ldr q0, [sp]
+; CHECK-NEXT:    str q0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
   %res.8 = load atomic i128, ptr %addr unordered, align 8
   store i128 %res.8, ptr %addr
 
   ret void
 }
 
-define void @test_nonfolded_load1(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load1:
-
-; CHECK: add x[[ADDR:[0-9]+]], x0, #4
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_4(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_4:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.1 = getelementptr i8,  ptr %addr, i32 4
   %res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
   store i128 %res.1, ptr %addr
@@ -86,14 +146,15 @@ define void @test_nonfolded_load1(ptr %addr) {
   ret void
 }
 
-define void @test_nonfolded_load2(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load2:
-
-; CHECK: add x[[ADDR:[0-9]+]], x0, #512
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_512(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_512:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add x8, x0, #512
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.1 = getelementptr i8,  ptr %addr, i32 512
   %res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
   store i128 %res.1, ptr %addr
@@ -101,14 +162,15 @@ define void @test_nonfolded_load2(ptr %addr) {
   ret void
 }
 
-define void @test_nonfolded_load3(ptr %addr) {
-; CHECK-LABEL: test_nonfolded_load3:
-
-; CHECK: sub x[[ADDR:[0-9]+]], x0, #520
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x[[ADDR]]]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+define void @test_nonfolded_load_offset_negative_520(ptr %addr) {
+; CHECK-LABEL: test_nonfolded_load_offset_negative_520:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub x8, x0, #520
+; CHECK-NEXT:    ldp x8, x9, [x8]
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    mov v0.d[1], x9
+; CHECK-NEXT:    str q0, [x0]
+; CHECK-NEXT:    ret
   %addr8.1 = getelementptr i8,  ptr %addr, i32 -520
   %res.1 = load atomic i128, ptr %addr8.1 monotonic, align 16
   store i128 %res.1, ptr %addr
@@ -116,34 +178,76 @@ define void @test_nonfolded_load3(ptr %addr) {
   ret void
 }
 
-define void @test_atomic_store(ptr %addr, i128 %val) {
-; CHECK-LABEL: test_atomic_store:
-
-; CHECK: stp x2, x3, [x0]
+define void @test_atomic_store_monotonic(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_monotonic:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    ret
   store atomic i128 %val, ptr %addr monotonic, align 16
 
-; CHECK: stp x2, x3, [x0]
+  ret void
+}
+
+define void @test_atomic_store_unordered(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_unordered:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    ret
   store atomic i128 %val, ptr %addr unordered, align 16
 
-; CHECK: dmb ish
-; CHECK: stp x2, x3, [x0]
+  ret void
+}
+
+define void @test_atomic_store_release(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_release:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dmb ish
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    ret
   store atomic i128 %val, ptr %addr release, align 16
 
-; CHECK: dmb ish
-; CHECK: stp x2, x3, [x0]
-; CHECK: dmb ish
+  ret void
+}
+
+define void @test_atomic_store_seq_cst(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_seq_cst:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    dmb ish
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    dmb ish
+; CHECK-NEXT:    ret
   store atomic i128 %val, ptr %addr seq_cst, align 16
 
+  ret void
+}
 
-; CHECK: stp x2, x3, [x0, #8]
+define void @test_atomic_store_offset_8(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x2, x3, [x0, #8]
+; CHECK-NEXT:    ret
   %addr8.1 = getelementptr i8,  ptr %addr, i32 8
   store atomic i128 %val, ptr %addr8.1 monotonic, align 16
 
-; CHECK: stp x2, x3, [x0, #504]
+  ret void
+}
+
+define void @test_atomic_store_offset_504(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_504:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x2, x3, [x0, #504]
+; CHECK-NEXT:    ret
   %addr8.2 = getelementptr i8,  ptr %addr, i32 504
   store atomic i128 %val, ptr %addr8.2 monotonic, align 16
 
-; CHECK: stp x2, x3, [x0, #-512]
+  ret void
+}
+
+define void @test_atomic_store_offset_negative_512(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_offset_negative_512:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x2, x3, [x0, #-512]
+; CHECK-NEXT:    ret
   %addr8.3 = getelementptr i8,  ptr %addr, i32 -512
   store atomic i...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/206936


More information about the llvm-commits mailing list