[llvm] [AArch64] fix 128-bit Sequentially Consistent load (PR #206936)
Tomas Matheson via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 06:37:18 PDT 2026
https://github.com/tommat01 updated https://github.com/llvm/llvm-project/pull/206936
>From c284566da0b726077d77a28c4eb4eca4cb349218 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson at arm.com>
Date: Wed, 1 Jul 2026 11:47:04 +0100
Subject: [PATCH 1/2] [AArch64][NFC] split up atomic tests
---
.../AArch64/GlobalISel/v8.4-atomic-128.ll | 80 +++++++++++++++++--
llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll | 78 ++++++++++++++++--
2 files changed, 148 insertions(+), 10 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
index fe7e24c2d8ba5..6b1dfdb9eb930 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
@@ -1,8 +1,8 @@
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+v8.4a %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse2 %s -o - -global-isel=1 -global-isel-abort=1 | FileCheck %s
-define void @test_atomic_load(ptr %addr) {
-; CHECK-LABEL: test_atomic_load:
+define void @test_atomic_load_128_monotonic(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic:
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
@@ -11,6 +11,12 @@ define void @test_atomic_load(ptr %addr) {
%res.0 = load atomic i128, ptr %addr monotonic, align 16
store i128 %res.0, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_unordered(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_unordered:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
; CHECK: mov v[[Q]].d[1], [[HI]]
@@ -18,6 +24,12 @@ define void @test_atomic_load(ptr %addr) {
%res.1 = load atomic i128, ptr %addr unordered, align 16
store i128 %res.1, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_acquire(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_acquire:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: dmb ish
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
@@ -26,14 +38,25 @@ define void @test_atomic_load(ptr %addr) {
%res.2 = load atomic i128, ptr %addr acquire, align 16
store i128 %res.2, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_128_seq_cst(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_seq_cst:
+
; CHECK: dmb ish
+; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
; CHECK: mov v[[Q]].d[1], [[HI]]
; CHECK: str q[[Q]], [x0]
%res.3 = load atomic i128, ptr %addr seq_cst, align 16
store i128 %res.3, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_monotonic_folded_offset_8(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_offset_8:
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #8]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
@@ -43,6 +66,12 @@ define void @test_atomic_load(ptr %addr) {
%res.5 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.5, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_monotonic_folded_max_offset(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_max_offset:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #504]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
; CHECK: mov v[[Q]].d[1], [[HI]]
@@ -51,6 +80,12 @@ define void @test_atomic_load(ptr %addr) {
%res.6 = load atomic i128, ptr %addr8.2 monotonic, align 16
store i128 %res.6, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_monotonic_folded_min_offset(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_min_offset:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #-512]
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
; CHECK: mov v[[Q]].d[1], [[HI]]
@@ -116,33 +151,68 @@ define void @test_nonfolded_load3(ptr %addr) {
ret void
}
-define void @test_atomic_store(ptr %addr, i128 %val) {
-; CHECK-LABEL: test_atomic_store:
+define void @test_atomic_store_128_monotonic(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic:
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_unordered(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_unordered:
+
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr unordered, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_release(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_release:
+
; CHECK: dmb ish
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr release, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_seq_cst(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_seq_cst:
+
; CHECK: dmb ish
; CHECK: stp x2, x3, [x0]
; CHECK: dmb ish
store atomic i128 %val, ptr %addr seq_cst, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_monotonic_folded_offset_8(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_offset_8:
; CHECK: stp x2, x3, [x0, #8]
%addr8.1 = getelementptr i8, ptr %addr, i32 8
store atomic i128 %val, ptr %addr8.1 monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_monotonic_folded_max_offset(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_max_offset:
+
; CHECK: stp x2, x3, [x0, #504]
%addr8.2 = getelementptr i8, ptr %addr, i32 504
store atomic i128 %val, ptr %addr8.2 monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_monotonic_folded_min_offset(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_min_offset:
+
; CHECK: stp x2, x3, [x0, #-512]
%addr8.3 = getelementptr i8, ptr %addr, i32 -512
store atomic i128 %val, ptr %addr8.3 monotonic, align 16
diff --git a/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
index 9988a74d4fd16..2297ecd9e9f3e 100644
--- a/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
@@ -1,32 +1,54 @@
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+v8.4a %s -o - | FileCheck %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+lse2 %s -o - | FileCheck %s
-define void @test_atomic_load(ptr %addr) {
-; CHECK-LABEL: test_atomic_load:
+define void @test_atomic_load_128_monotonic(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic:
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: stp [[LO]], [[HI]], [x0]
%res.0 = load atomic i128, ptr %addr monotonic, align 16
store i128 %res.0, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_unordered(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_unordered:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: stp [[LO]], [[HI]], [x0]
%res.1 = load atomic i128, ptr %addr unordered, align 16
store i128 %res.1, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_acquire(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_acquire:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: dmb ish
; CHECK: stp [[LO]], [[HI]], [x0]
%res.2 = load atomic i128, ptr %addr acquire, align 16
store i128 %res.2, ptr %addr
-; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
+ ret void
+}
+
+define void @test_atomic_load_128_seq_cst(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_seq_cst:
+
; CHECK: dmb ish
+; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: stp [[LO]], [[HI]], [x0]
%res.3 = load atomic i128, ptr %addr seq_cst, align 16
store i128 %res.3, ptr %addr
+ ret void
+}
+define void @test_atomic_load_128_monotonic_folded_offset_32(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_offset_32:
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #32]
; CHECK-DAG: stp [[LO]], [[HI]], [x0]
@@ -34,12 +56,24 @@ define void @test_atomic_load(ptr %addr) {
%res.5 = load atomic i128, ptr %addr8.1 monotonic, align 16
store i128 %res.5, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_monotonic_folded_max_offset(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_max_offset:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #504]
; CHECK: stp [[LO]], [[HI]], [x0]
%addr8.2 = getelementptr i8, ptr %addr, i32 504
%res.6 = load atomic i128, ptr %addr8.2 monotonic, align 16
store i128 %res.6, ptr %addr
+ ret void
+}
+
+define void @test_atomic_load_128_monotonic_folded_min_offset(ptr %addr) {
+; CHECK-LABEL: test_atomic_load_128_monotonic_folded_min_offset:
+
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0, #-512]
; CHECK: stp [[LO]], [[HI]], [x0]
%addr8.3 = getelementptr i8, ptr %addr, i32 -512
@@ -97,34 +131,68 @@ define void @test_nonfolded_load3(ptr %addr) {
ret void
}
-define void @test_atomic_store(ptr %addr, i128 %val) {
-; CHECK-LABEL: test_atomic_store:
+define void @test_atomic_store_128_monotonic(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic:
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_unordered(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_unordered:
+
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr unordered, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_release(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_release:
+
; CHECK: dmb ish
; CHECK: stp x2, x3, [x0]
store atomic i128 %val, ptr %addr release, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_seq_cst(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_seq_cst:
+
; CHECK: dmb ish
; CHECK: stp x2, x3, [x0]
; CHECK: dmb ish
store atomic i128 %val, ptr %addr seq_cst, align 16
+ ret void
+}
+define void @test_atomic_store_128_monotonic_folded_offset_8(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_offset_8:
; CHECK: stp x2, x3, [x0, #8]
%addr8.1 = getelementptr i8, ptr %addr, i32 8
store atomic i128 %val, ptr %addr8.1 monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_monotonic_folded_max_offset(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_max_offset:
+
; CHECK: stp x2, x3, [x0, #504]
%addr8.2 = getelementptr i8, ptr %addr, i32 504
store atomic i128 %val, ptr %addr8.2 monotonic, align 16
+ ret void
+}
+
+define void @test_atomic_store_128_monotonic_folded_min_offset(ptr %addr, i128 %val) {
+; CHECK-LABEL: test_atomic_store_128_monotonic_folded_min_offset:
+
; CHECK: stp x2, x3, [x0, #-512]
%addr8.3 = getelementptr i8, ptr %addr, i32 -512
store atomic i128 %val, ptr %addr8.3 monotonic, align 16
>From 21738a41bb4301decb323e24ca4327764ba48793 Mon Sep 17 00:00:00 2001
From: Tomas Matheson <tomas.matheson at arm.com>
Date: Wed, 1 Jul 2026 11:39:21 +0100
Subject: [PATCH 2/2] [AArch64] emit fence before 128-bit SC atomic load
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 12 ++++++++++++
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 ++
.../AArch64/Atomics/aarch64-atomic-load-lse2.ll | 2 ++
.../Atomics/aarch64-atomic-load-lse2_lse128.ll | 2 ++
.../AArch64/Atomics/aarch64-atomic-load-rcpc3.ll | 2 ++
.../AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll | 2 ++
.../AArch64/Atomics/aarch64_be-atomic-load-lse2.ll | 2 ++
.../Atomics/aarch64_be-atomic-load-lse2_lse128.ll | 2 ++
.../AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll | 2 ++
.../CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll | 1 +
llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll | 1 +
11 files changed, 30 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c4c52dbe395e8..355627228a246 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -31723,6 +31723,18 @@ bool AArch64TargetLowering::shouldInsertTrailingSeqCstFenceForAtomicStore(
return !Subtarget->hasLSE();
}
+Instruction *AArch64TargetLowering::emitLeadingFence(IRBuilderBase &Builder,
+ Instruction *Inst,
+ AtomicOrdering Ord) const {
+ // Keep seq_cst 128-bit LSE2 loads ordered against v8.0-style seq_cst LL/SC
+ // stores by placing the full barrier before the single-copy atomic load.
+ if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent &&
+ isOpSuitableForLDPSTP(Inst))
+ return Builder.CreateFence(Ord);
+
+ return TargetLoweringBase::emitLeadingFence(Builder, Inst, Ord);
+}
+
// Loads and stores less than 128-bits are already atomic; ones above that
// are doomed anyway, so defer to the default libcall and blame the OS when
// things go wrong.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 1341a4b3e03b6..69013d4010122 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -346,6 +346,8 @@ class AArch64TargetLowering : public TargetLowering {
bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context,
EVT VT) const override;
+ Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
+ AtomicOrdering Ord) const override;
Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
AtomicOrdering Ord) const override;
Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
index 0e9c29758244a..c8d6ccd473f27 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2.ll
@@ -273,6 +273,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -281,6 +282,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
index 2f030b0e23ab1..89d62ed07039b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-lse2_lse128.ll
@@ -273,6 +273,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -281,6 +282,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
index 435f0a0c01765..a2ce401c8b71b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc3.ll
@@ -271,6 +271,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -279,6 +280,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
index f91ab9fa3e8ca..17e779088ece7 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-load-rcpc_immo.ll
@@ -383,6 +383,7 @@ define i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) {
define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0, #64]
; CHECK: dmb ish
%gep = getelementptr inbounds i128, ptr %ptr, i32 4
@@ -392,6 +393,7 @@ define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0, #64]
; CHECK: dmb ish
%gep = getelementptr inbounds i128, ptr %ptr, i32 4
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
index 08f7dd32c1280..206a1e6236334 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2.ll
@@ -273,6 +273,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -281,6 +282,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
index 74e612d59858f..bd329a1d6692d 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-lse2_lse128.ll
@@ -273,6 +273,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -281,6 +282,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
index de83b702d988a..5a859e3146cd4 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomic-load-rcpc3.ll
@@ -271,6 +271,7 @@ define dso_local i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr)
define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
@@ -279,6 +280,7 @@ define dso_local i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
define dso_local i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
+; CHECK: dmb ish
; CHECK: ldp x0, x1, [x0]
; CHECK: dmb ish
%r = load atomic i128, ptr %ptr seq_cst, align 16
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
index 6b1dfdb9eb930..4299a51d1ae43 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
@@ -46,6 +46,7 @@ define void @test_atomic_load_128_seq_cst(ptr %addr) {
; CHECK: dmb ish
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
+; CHECK: dmb ish
; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
; CHECK: mov v[[Q]].d[1], [[HI]]
; CHECK: str q[[Q]], [x0]
diff --git a/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
index 2297ecd9e9f3e..e09b803a783cd 100644
--- a/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/v8.4-atomic-128.ll
@@ -40,6 +40,7 @@ define void @test_atomic_load_128_seq_cst(ptr %addr) {
; CHECK: dmb ish
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
+; CHECK: dmb ish
; CHECK: stp [[LO]], [[HI]], [x0]
%res.3 = load atomic i128, ptr %addr seq_cst, align 16
store i128 %res.3, ptr %addr
More information about the llvm-commits
mailing list