[clang] [llvm] [AArch64] Implement the atomic fetch with hint intrinsic (PR #227711)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Oct 13 03:24:30 PDT 2026
https://github.com/Lukacma updated https://github.com/llvm/llvm-project/pull/227711
>From ea8313ea0030b8838d8b212b9771d025ffd11b3c Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 16 Sep 2026 12:37:14 +0000
Subject: [PATCH 01/11] [AArch64] Add CMH hints to store_with_hint intrinsic
---
clang/lib/Headers/arm_acle.h | 3 +
clang/test/CodeGen/builtins-arm64.c | 19 ++++-
clang/test/Sema/builtins-arm64.c | 3 +-
.../include/llvm/Support/AArch64MemoryHints.h | 11 ++-
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 18 +++++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 9 +++
.../Atomics/aarch64-atomic-store-hint.ll | 80 ++++++++++++++++++-
.../Atomics/aarch64-relaxed-store-hint.ll | 18 +++--
8 files changed, 150 insertions(+), 11 deletions(-)
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 715fe851b0c32..395b760e7dbf0 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -746,6 +746,9 @@ __arm_st64bv0(void *__addr, data512_t __value) {
#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
#define HINT_STSHH_KEEP 0
#define HINT_STSHH_STRM 1
+#define HINT_STCPH 2
+#define HINT_SHUH 3
+#define HINT_SHUH_PH 4
#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
__builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index fb297cd3eec2c..9c3d810d55974 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -230,9 +230,18 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM);
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M5:[0-9]+]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_SHUH);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M7:[0-9]+]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_SHUH_PH);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M9:[0-9]+]]
+
// Invalid hint should be dropped
- __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}}
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
@@ -240,3 +249,9 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0}
// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
// CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1}
+// CHECK: ![[M5]] = !{i32 1, ![[M6:[0-9]+]]}
+// CHECK: ![[M6]] = !{!"aarch64.mem_hint", i32 2}
+// CHECK: ![[M7]] = !{i32 1, ![[M8:[0-9]+]]}
+// CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3}
+// CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]}
+// CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 9a6e0715c4c78..a79fb9177cec8 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -73,7 +73,8 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}}
+
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
index b917191499cac..f47eb0c945d1b 100644
--- a/llvm/include/llvm/Support/AArch64MemoryHints.h
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -14,11 +14,14 @@ enum class AArch64MemoryHint {
NONE = 0,
STSHH_KEEP = 1,
STSHH_STRM = 2,
+ STCPH = 3,
+ SHUH = 4,
+ SHUH_PH = 5,
};
template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
return (Int)AArch64MemoryHint::STSHH_KEEP <= I &&
- I <= (Int)AArch64MemoryHint::STSHH_STRM;
+ I <= (Int)AArch64MemoryHint::SHUH_PH;
}
template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
@@ -27,6 +30,12 @@ template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
return AArch64MemoryHint::STSHH_KEEP;
case 1:
return AArch64MemoryHint::STSHH_STRM;
+ case 2:
+ return AArch64MemoryHint::STCPH;
+ case 3:
+ return AArch64MemoryHint::SHUH;
+ case 4:
+ return AArch64MemoryHint::SHUH_PH;
default:
return AArch64MemoryHint::NONE;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 48c0e37151b9c..aa07673673ce2 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -520,6 +520,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
bool isAtomicSTSHH_KEEP(SDNode *N) const;
bool isAtomicSTSHH_STRM(SDNode *N) const;
+ bool isAtomicSTCPH(SDNode *N) const;
+ bool isAtomicSHUH(SDNode *N) const;
+ bool isAtomicSHUH_PH(SDNode *N) const;
bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
bool Negate);
@@ -4647,6 +4650,21 @@ bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
AArch64MemoryHint::STSHH_STRM;
}
+bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::STCPH;
+}
+
+bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::SHUH;
+}
+
+bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::SHUH_PH;
+}
+
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
SDValue &Shift, bool Negate) {
if (!isa<ConstantSDNode>(N))
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 7e0e75c7a3c6b..56e4821fc20b8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -425,11 +425,20 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
let AddedComplexity = 15 in {
defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>;
+ defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>;
+ defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>;
defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>;
+ defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>;
+ defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>;
+ defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>;
+ defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>;
+ defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>;
}
//===----------------------------------
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index bfa9fc49cf752..b015d020dc07f 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -318,6 +318,78 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw
ret void
}
+;
+; STCPH
+;
+
+define void @test_atomic_store_stcph_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stcph_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stcph
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !4
+ ret void
+}
+
+define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stcph_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stcph
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !4
+ ret void
+}
+
+;
+; SHUH
+;
+
+define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6
+ ret void
+}
+
+define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6
+ ret void
+}
+
+;
+; SHUH PH
+;
+
+define void @test_atomic_store_shuh_ph_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_ph_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !8
+ ret void
+}
+
+define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_ph_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !8
+ ret void
+}
+
;
; Invalid Hint
;
@@ -327,7 +399,7 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !10
ret void
}
@@ -337,3 +409,9 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
!3 = !{!"aarch64.mem_hint", i32 1}
!4 = !{i32 1, !5}
!5 = !{!"aarch64.mem_hint", i32 2}
+!6 = !{i32 1, !7}
+!7 = !{!"aarch64.mem_hint", i32 3}
+!8 = !{i32 1, !9}
+!9 = !{!"aarch64.mem_hint", i32 4}
+!10 = !{i32 1, !11}
+!11 = !{!"aarch64.mem_hint", i32 5}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index 5f5bc43039778..b862ae7f4a15d 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -11,11 +11,11 @@
define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stcph
; CHECK-NEXT: strb w2, [x0, w1, sxtw]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !4
ret void
}
@@ -124,11 +124,11 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n
define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: stshh strm
+; CHECK-NEXT: shuh{{$}}
; CHECK-NEXT: strb w2, [x0, x1]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i64 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !6
ret void
}
@@ -237,11 +237,11 @@ define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) n
define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: stshh strm
+; CHECK-NEXT: shuh ph
; CHECK-NEXT: strb w1, [x0, #4095]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 4095
- store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !8
ret void
}
@@ -460,3 +460,9 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
!1 = !{!"aarch64.mem_hint", i32 0}
!2 = !{i32 1, !3}
!3 = !{!"aarch64.mem_hint", i32 1}
+!4 = !{i32 1, !5}
+!5 = !{!"aarch64.mem_hint", i32 2}
+!6 = !{i32 1, !7}
+!7 = !{!"aarch64.mem_hint", i32 3}
+!8 = !{i32 1, !9}
+!9 = !{!"aarch64.mem_hint", i32 4}
>From 9181f1536f6916aa58893abad6ba0a1dba0c323a Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 30 Sep 2026 11:39:27 +0000
Subject: [PATCH 02/11] refactor isAtomic* functions and address review
comments
---
clang/test/CodeGen/builtins-arm64.c | 2 +-
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 +++----------------
.../lib/Target/AArch64/AArch64InstrAtomics.td | 32 +++++++++----------
.../Atomics/aarch64-atomic-store-hint.ll | 30 +++++++++++++++++
.../Atomics/aarch64-relaxed-store-hint.ll | 2 +-
5 files changed, 52 insertions(+), 46 deletions(-)
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 9c3d810d55974..b100236258580 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -241,7 +241,7 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8{{$}}
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index aa07673673ce2..2416936309923 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -518,11 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
- bool isAtomicSTSHH_KEEP(SDNode *N) const;
- bool isAtomicSTSHH_STRM(SDNode *N) const;
- bool isAtomicSTCPH(SDNode *N) const;
- bool isAtomicSHUH(SDNode *N) const;
- bool isAtomicSHUH_PH(SDNode *N) const;
+ bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const;
bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
bool Negate);
@@ -4640,29 +4636,9 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
return toAArch64MemoryHint(MemoryHint);
}
-bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
- return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64MemoryHint::STSHH_KEEP;
-}
-
-bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
- return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64MemoryHint::STSHH_STRM;
-}
-
-bool AArch64DAGToDAGISel::isAtomicSTCPH(SDNode *N) const {
- return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64MemoryHint::STCPH;
-}
-
-bool AArch64DAGToDAGISel::isAtomicSHUH(SDNode *N) const {
- return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64MemoryHint::SHUH;
-}
-
-bool AArch64DAGToDAGISel::isAtomicSHUH_PH(SDNode *N) const {
- return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64MemoryHint::SHUH_PH;
+bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N,
+ AArch64MemoryHint Hint) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint;
}
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 56e4821fc20b8..09464116b2bfb 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -423,22 +423,22 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicSTCPH(N); }]>;
- defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicSHUH(N); }]>;
- defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicSHUH_PH(N); }]>;
-
- defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<1, 52, [{ return isAtomicSTCPH(N); }]>;
- defm : AtomicHintPatterns<0, 52, [{ return isAtomicSTCPH(N); }]>;
- defm : AtomicHintPatterns<1, 50, [{ return isAtomicSHUH(N); }]>;
- defm : AtomicHintPatterns<0, 50, [{ return isAtomicSHUH(N); }]>;
- defm : AtomicHintPatterns<1, 51, [{ return isAtomicSHUH_PH(N); }]>;
- defm : AtomicHintPatterns<0, 51, [{ return isAtomicSHUH_PH(N); }]>;
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>;
+ defm : AtomicHintPatternsAddrMode<52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>;
+ defm : AtomicHintPatternsAddrMode<50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>;
+ defm : AtomicHintPatternsAddrMode<51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>;
+
+ defm : AtomicHintPatterns<1, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>;
+ defm : AtomicHintPatterns<0, 48, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_KEEP); }]>;
+ defm : AtomicHintPatterns<1, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>;
+ defm : AtomicHintPatterns<0, 49, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STSHH_STRM); }]>;
+ defm : AtomicHintPatterns<1, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>;
+ defm : AtomicHintPatterns<0, 52, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::STCPH); }]>;
+ defm : AtomicHintPatterns<1, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>;
+ defm : AtomicHintPatterns<0, 50, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>;
+ defm : AtomicHintPatterns<1, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>;
+ defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>;
}
//===----------------------------------
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index b015d020dc07f..c989a5e40e091 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -342,6 +342,16 @@ define void @test_atomic_store_stcph_release_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
+define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stcph_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stcph
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !4
+ ret void
+}
+
;
; SHUH
;
@@ -366,6 +376,16 @@ define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
+define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !6
+ ret void
+}
+
;
; SHUH PH
;
@@ -390,6 +410,16 @@ define void @test_atomic_store_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind
ret void
}
+define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_shuh_ph_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !8
+ ret void
+}
+
;
; Invalid Hint
;
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index b862ae7f4a15d..74d50efa988bd 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -124,7 +124,7 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n
define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: strb w2, [x0, x1]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i64 %offset
>From 52d0edcdfbd5ec15dc236be83bd56fe8e9ff35ea Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 30 Sep 2026 11:44:39 +0000
Subject: [PATCH 03/11] formatting
---
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 2416936309923..43d3027d5bafa 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -4637,7 +4637,7 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
}
bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N,
- AArch64MemoryHint Hint) const {
+ AArch64MemoryHint Hint) const {
return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint;
}
>From 201de51d24648d961d02490f497635aed95eff6e Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 30 Sep 2026 12:24:37 +0000
Subject: [PATCH 04/11] [AArch64] Implement the atomic fetch with hint
intrinsic
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +
.../clang/Basic/DiagnosticSemaKinds.td | 4 +-
clang/include/clang/Sema/SemaARM.h | 2 +-
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 94 ++++++++--
clang/lib/Headers/arm_acle.h | 12 +-
clang/lib/Sema/SemaARM.cpp | 51 +++--
.../AArch64/atomic-store-hint-template.cpp | 31 +++
clang/test/CodeGen/arm_acle.c | 111 +++++++++++
clang/test/CodeGen/builtins-arm64.c | 53 ++++++
clang/test/Sema/builtins-arm64.c | 30 +++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 21 +++
.../Target/AArch64/AArch64ISelLowering.cpp | 6 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 75 +++++++-
.../Atomics/aarch64-atomic-store-hint.ll | 176 +++++++++++++++++-
14 files changed, 629 insertions(+), 42 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 30aa3d526cbbb..39ea7102b5d65 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -173,6 +173,11 @@ let Attributes = [NoThrow], Features = "ls64" in {
let Attributes = [NoThrow, CustomTypeChecking] in {
def atomic_store_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_add_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_sub_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_and_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_xor_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_or_with_hint : AArch64Builtin<"void(...)">;
}
// Armv9.3-A Guarded Control Stack
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index d293a9798da6a..f1c9b7e64f87f 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9618,7 +9618,7 @@ def err_atomic_builtin_must_be_pointer_intptr : Error<
def err_atomic_hint_builtin_must_be_pointer : Error<
"address argument to atomic hint builtin must be a pointer to a scalar "
- "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+ "%select{integral|integral or floating-point}1 type of 8, 16, 32, or 64 bits (%0 invalid)">;
def err_atomic_hint_has_invalid_hint_type : Error<
"invalid hint type argument to atomic hint operation (%0)">;
@@ -9691,7 +9691,7 @@ def err_atomic_op_needs_atomic_fp
: Error<"address argument to atomic operation must be a pointer to "
"%select{|atomic }0floating point type (%1 invalid)">;
def err_atomic_op_hint_data_size : Error<
- "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
+ "address argument to atomic %select{fetch|store}0 with hint must be of size 8, 16, 32 or 64 bits">;
def err_atomic_hint_has_invalid_memory_order : Error<
"invalid memory order argument to atomic hint operation (%0 invalid)">;
def warn_atomic_hint_has_invalid_hint_type : Warning<
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index b0a01c40ffece..76a06f4f4221a 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -70,7 +70,7 @@ class SemaARM : public SemaBase {
bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum,
unsigned ExpectedFieldNum, bool AllowName);
bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
- bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall);
bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index e20fa60ee132b..3293c263527d4 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2066,6 +2066,29 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
Ops);
}
+static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
+ Instruction *Atomic,
+ unsigned PtrOperand) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ LLVMContext &Ctx = CGF.CGM.getLLVMContext();
+ Expr::EvalResult Result;
+ if (!E->getArg(3)->EvaluateAsInt(Result, CGF.getContext()))
+ llvm_unreachable(
+ "Expected integer hint argument to atomic operation with hint.");
+ unsigned HintArg = Result.Val.getInt().getExtValue();
+
+ if (toAArch64MemoryHint(HintArg) == AArch64MemoryHint::NONE)
+ return;
+
+ MDNode *MemHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
+ MDNode *HintNode = MDNode::get(
+ Ctx, {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)),
+ MemHint});
+ Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
+}
+
static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned BuiltinID,
const CallExpr *E) {
@@ -2105,26 +2128,53 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
break;
}
Store->setAtomic(Ordering);
+ AttachAtomicHintMetadata(CGF, E, Store, 1);
+ return Store;
+}
- if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
+static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF,
+ const CallExpr *E,
+ AtomicRMWInst::BinOp Op) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ Expr::EvalResult Result;
+ if (!E->getArg(2)->EvaluateAsInt(Result, CGF.getContext()))
llvm_unreachable(
- "Expected integer hint argument to atomic store with hint.");
- unsigned HintArg = Result.Val.getInt().getExtValue();
+ "Expected integer policy argument to atomic fetch with hint.");
- // Attach the hint if valid
- if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::NONE) {
- LLVMContext &Ctx = CGM.getLLVMContext();
- MDNode *MemHint = MDNode::get(
- Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
- MDNode *HintNode = MDNode::get(
- CGM.getLLVMContext(),
- {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
+ AtomicOrdering Ordering;
+ unsigned OrderingArg = Result.Val.getInt().getExtValue();
+ assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
- Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
+ switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
+ case AtomicOrderingCABI::relaxed:
+ Ordering = AtomicOrdering::Monotonic;
+ break;
+ case AtomicOrderingCABI::consume:
+ case AtomicOrderingCABI::acquire:
+ Ordering = AtomicOrdering::Acquire;
+ break;
+ case AtomicOrderingCABI::release:
+ Ordering = AtomicOrdering::Release;
+ break;
+ case AtomicOrderingCABI::acq_rel:
+ Ordering = AtomicOrdering::AcquireRelease;
+ break;
+ case AtomicOrderingCABI::seq_cst:
+ Ordering = AtomicOrdering::SequentiallyConsistent;
+ break;
}
- return Store;
+ const Expr *Ptr = E->getArg(0);
+ Address Addr = CGF.EmitPointerWithAlignment(Ptr);
+ Addr = Addr.withElementType(
+ CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
+ const Expr *Data = E->getArg(1);
+ Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
+
+ AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, DataVal, Ordering);
+ RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
+ AttachAtomicHintMetadata(CGF, E, RMW, 0);
+ return CGF.EmitFromMemory(RMW, E->getType());
}
/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
@@ -4902,8 +4952,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
- if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ switch (BuiltinID) {
+ default:
+ break;
+ case AArch64::BI__builtin_arm_atomic_store_with_hint:
return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
+ case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Add);
+ case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Sub);
+ case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::And);
+ case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Xor);
+ case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Or);
+ }
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 395b760e7dbf0..cfbe490d995bf 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -742,7 +742,7 @@ __arm_st64bv0(void *__addr, data512_t __value) {
}
#endif
-/* Atomic store with hints */
+/* Atomic intrinsics with hints */
#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
#define HINT_STSHH_KEEP 0
#define HINT_STSHH_STRM 1
@@ -751,6 +751,16 @@ __arm_st64bv0(void *__addr, data512_t __value) {
#define HINT_SHUH_PH 4
#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
__builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint)
#endif
/* 11.1 Special register intrinsics */
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index a5037fce03c6b..2f5a9f985a81d 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -323,14 +323,16 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
return false;
}
-bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
- CallExpr *TheCall) {
+bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) {
if (SemaRef.checkArgCount(TheCall, 4))
return true;
- // Arg 0 should be the pointer type. The pointee type must be a
- // scalar integral or floating-point type of 8, 16, 32 or 64 bits.
+ bool IsStore = BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint;
ASTContext &Context = getASTContext();
+
+ // Arg 0 should be the pointer type. The pointee type must be a
+ // scalar integral type of 8, 16, 32 or 64 bits. Stores also support
+ // floating-point types.
auto PtrArgRes =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
if (PtrArgRes.isInvalid())
@@ -340,7 +342,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_builtin_must_be_pointer)
- << PtrArg->getType() << 0 << PtrArg->getSourceRange();
+ << PtrArg->getType() << IsStore << PtrArg->getSourceRange();
TheCall->setArg(0, PtrArg);
QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType());
@@ -350,10 +352,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrQT << PtrArg->getSourceRange();
PtrQT = PtrQT.getUnqualifiedType();
- if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
- !PtrQT->isMFloat8Type())
+ if (!PtrQT->isIntegralType(Context) &&
+ !(IsStore && (PtrQT->isFloatingType() || PtrQT->isMFloat8Type())))
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_op_needs_atomic_int_or_fp)
+ IsStore ? diag::err_atomic_op_needs_atomic_int_or_fp
+ : diag::err_atomic_op_needs_atomic_int)
<< 0 << PtrQT << PtrArg->getSourceRange();
if (PtrQT->isBitIntType())
@@ -364,9 +367,12 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
- << PtrArg->getSourceRange();
+ << IsStore << PtrArg->getSourceRange();
+
+ if (!IsStore)
+ TheCall->setType(PtrQT);
- // Arg 1 is the data to be stored. The type must match the pointee
+ // Arg 1 is the data operand. The type must match the pointee
// type found above.
auto DataArgRes =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
@@ -382,7 +388,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_typecheck_call_different_arg_types)
<< PtrQT << DataQT;
- // Arg 2 is the memory order, which must be relaxed, release or seq_cst
+ // Arg 2 is the memory order. Stores only allow relaxed, release or seq_cst.
auto MemOrdArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2));
if (MemOrdArg.isInvalid())
@@ -407,7 +413,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< *MemOrdAP << MemOrd->getSourceRange();
auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
- if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ if (IsStore && AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
AtomicOrdering != llvm::AtomicOrderingCABI::release &&
AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
return Diag(TheCall->getBeginLoc(),
@@ -416,7 +422,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
}
// Arg 3 is the hint type. Only values represented by AArch64MemoryHint
- // are valid.
+ // are valid. Fetch operations only allow HINT_SHUH and HINT_SHUH_PH.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3));
if (HintArg.isInvalid())
@@ -433,8 +439,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_hint_has_invalid_hint_type)
<< Hint->getType() << Hint->getSourceRange();
- if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) ==
- llvm::AArch64MemoryHint::NONE) {
+ auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue());
+ if (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH &&
+ MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << *HintAP << Hint->getSourceRange();
+
+ if (MemoryHint == llvm::AArch64MemoryHint::NONE) {
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << Hint->getSourceRange();
return false;
@@ -1291,8 +1303,13 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
BuiltinID == AArch64::BI__builtin_arm_wsrp)
return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
- if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
- return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_add_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_and_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_or_with_hint)
+ return BuiltinARMAtomicHintCall(BuiltinID, TheCall);
// Only check the valid encoding range. Any constant in this range would be
// converted to a register of the form S2_2_C3_C4_5. Let the hardware throw
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
index ecf1e10a09b5d..17e99be63adcd 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -55,9 +55,40 @@ void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) {
__builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{});
}
+template <unsigned Order>
+int test_atomic_fetch_hint_template_order(int *ptr, int val) {
+ return __arm_atomic_fetch_add_with_hint(ptr, val, Order, 3);
+}
+
+template int test_atomic_fetch_hint_template_order<__ATOMIC_ACQUIRE>(int *, int);
+
+template <unsigned Hint>
+int test_atomic_fetch_hint_template_hint(int *ptr, int val) {
+ return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
+}
+
+template int test_atomic_fetch_hint_template_hint<4>(int *, int);
+
+// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]]
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
+//
+// CHECK-LABEL: define weak_odr noundef i32 @_Z36test_atomic_fetch_hint_template_hintILj4EEiPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw xor ptr [[PTR]], i32 [[VAL]] seq_cst, align 4, !mem.cache_hint [[META10:![0-9]+]]
+// CHECK-NEXT: ret i32 [[TMP0]]
+//
//.
// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
// CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]}
// CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1}
+// CHECK: [[META8]] = !{i32 0, [[META9:![0-9]+]]}
+// CHECK: [[META9]] = !{!"aarch64.mem_hint", i32 3}
+// CHECK: [[META10]] = !{i32 0, [[META11:![0-9]+]]}
+// CHECK: [[META11]] = !{!"aarch64.mem_hint", i32 4}
//.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 60ec3e48e9498..2acb0373632c5 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1989,9 +1989,120 @@ void test_atomic_store_hint_array_arg() {
__builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM);
}
+// AArch64-LABEL: @test_atomic_fetch_hint_char(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[DATA:%.*]] monotonic, align 1, !mem.cache_hint [[FETCH0:![0-9]+]]
+// AArch64-NEXT: ret i8 [[OLD]]
+//
+char test_atomic_fetch_hint_char(char *ptr, char data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_uchar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[DATA:%.*]] release, align 1, !mem.cache_hint [[FETCH1:![0-9]+]]
+// AArch64-NEXT: ret i8 [[OLD]]
+//
+unsigned char test_atomic_fetch_hint_uchar(unsigned char *ptr, unsigned char data) {
+ return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELEASE, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_short(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[DATA:%.*]] acquire, align 2, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i16 [[OLD]]
+//
+short test_atomic_fetch_hint_short(short *ptr, short data) {
+ return __arm_atomic_fetch_and_with_hint(ptr, data, __ATOMIC_ACQUIRE, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_ushort(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[DATA:%.*]] acq_rel, align 2, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i16 [[OLD]]
+//
+unsigned short test_atomic_fetch_hint_ushort(unsigned short *ptr, unsigned short data) {
+ return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_ACQ_REL, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[DATA:%.*]] seq_cst, align 4, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_int(int *ptr, int data) {
+ return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_SEQ_CST, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_unsigned(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] acquire, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+unsigned test_atomic_fetch_hint_unsigned(unsigned *ptr, unsigned data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_CONSUME, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_long(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[DATA:%.*]] monotonic, align 8, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i64 [[OLD]]
+//
+long test_atomic_fetch_hint_long(long *ptr, long data) {
+ return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_bool(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8
+// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0
+// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[STOREDV1]] monotonic, align 1, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: [[LOADEDV1:%.*]] = icmp ne i8 [[OLD]], 0
+// AArch64-NEXT: ret i1 [[LOADEDV1]]
+//
+bool test_atomic_fetch_hint_bool(bool *ptr, bool data) {
+ return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_volatile(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw volatile or ptr [[PTR:%.*]], i32 [[DATA:%.*]] acq_rel, align 4, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_volatile(volatile int *ptr, int data) {
+ return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_ACQ_REL, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_typedef(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] release, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+aliased_int test_atomic_fetch_hint_typedef(aliased_int *ptr, const int data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELEASE, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_array_arg(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4
+// AArch64-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[STORAGE]], i8 0, i64 4, i1 false)
+// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[ARRAYDECAY]], i32 1 monotonic, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_array_arg(void) {
+ int storage[1] = {0};
+ return __arm_atomic_fetch_sub_with_hint(storage, 1, __ATOMIC_RELAXED, 4);
+}
+
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1}
+// AArch64: [[FETCH0]] = !{i32 0, [[FETCH_HINT0:![0-9]+]]}
+// AArch64-NEXT: [[FETCH_HINT0]] = !{!"aarch64.mem_hint", i32 3}
+// AArch64-NEXT: [[FETCH1]] = !{i32 0, [[FETCH_HINT1:![0-9]+]]}
+// AArch64-NEXT: [[FETCH_HINT1]] = !{!"aarch64.mem_hint", i32 4}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index b100236258580..33960feb6cdb2 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -242,6 +242,57 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
+ // CHECK-NOT: !mem.cache_hint
+ // CHECK-NEXT: ret void
+}
+
+// CHECK-LABEL: @atomic_fetch_add_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F1:[0-9]+]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_add_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_sub_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw sub ptr {{.*}}, i64 {{.*}} release, align 8, !mem.cache_hint ![[F3:[0-9]+]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_sub_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELEASE, 4);
+}
+
+// CHECK-LABEL: @atomic_fetch_and_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw and ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_and_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_ACQUIRE, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_xor_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw xor ptr {{.*}}, i64 {{.*}} acq_rel, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_xor_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_xor_with_hint(a, b, __ATOMIC_ACQ_REL, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_or_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw or ptr {{.*}}, i64 {{.*}} seq_cst, align 8, !mem.cache_hint ![[F3]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_or_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_or_with_hint(a, b, __ATOMIC_SEQ_CST, 4);
+}
+
+// CHECK-LABEL: @atomic_fetch_with_hint_consume(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_with_hint_consume(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_CONSUME, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_add_with_hint_ph(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F3]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4);
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
@@ -255,3 +306,5 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3}
// CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]}
// CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4}
+// CHECK: ![[F1]] = !{i32 0, ![[M8]]}
+// CHECK: ![[F3]] = !{i32 0, ![[M10]]}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index a79fb9177cec8..ba0cf233df319 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -80,3 +80,33 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
// expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
}
+
+void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr,
+ float *f_ptr, float f_data, __int128 *wide_ptr,
+ __int128 wide_data, unsigned _BitInt(8) *bit_ptr,
+ unsigned _BitInt(8) bit_data, int variable,
+ int **ptr_ptr, _Atomic int *atomic_ptr) {
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 3, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+
+ __builtin_arm_atomic_fetch_and_with_hint(0, c_data, 0, 3); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral type of 8, 16, 32, or 64 bits ('int' invalid)}}
+ __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, f_data, 0, 3); // expected-error {{arguments are of different types ('char' vs 'float')}}
+ __builtin_arm_atomic_fetch_or_with_hint(const_c_ptr, c_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
+ __builtin_arm_atomic_fetch_add_with_hint(f_ptr, f_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('float' invalid)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(wide_ptr, wide_data, 0, 3); // expected-error {{address argument to atomic fetch with hint must be of size 8, 16, 32 or 64 bits}}
+ __builtin_arm_atomic_fetch_and_with_hint(bit_ptr, bit_data, 0, 3); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+ __builtin_arm_atomic_fetch_xor_with_hint(ptr_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('int *' invalid)}}
+ __builtin_arm_atomic_fetch_or_with_hint(atomic_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('_Atomic(int)' invalid)}}
+
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, variable, 3); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, -1, 3); // expected-error {{invalid memory order argument to atomic hint operation (-1 invalid)}}
+ __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 6, 3); // expected-error {{invalid memory order argument to atomic hint operation (6 invalid)}}
+
+ __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
+ // expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-error {{invalid hint type argument to atomic hint operation (0)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-error {{invalid hint type argument to atomic hint operation (1)}}
+ __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-error {{invalid hint type argument to atomic hint operation (2)}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-error {{invalid hint type argument to atomic hint operation (5)}}
+}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 5a502f785e0f4..73e1a0e9b3a52 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -295,6 +295,9 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
+ // Emit expansion of atomic fetch with hint pseudo instructions
+ void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI);
+
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
EmitToStreamer(*OutStreamer, Inst);
@@ -3409,6 +3412,20 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
EmitToStreamer(*OutStreamer, Store);
}
+void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion(
+ const MachineInstr *MI) {
+ EmitToStreamer(
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
+
+ MCInst Fetch;
+ Fetch.setOpcode(MI->getOperand(3).getImm());
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(2).getReg()));
+ Fetch.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Fetch);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -4132,6 +4149,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_X:
emitAtomicHintPseudoExpansion(MI);
return;
+ case AArch64::ATOMIC_FETCH_HINT_W:
+ case AArch64::ATOMIC_FETCH_HINT_X:
+ emitAtomicFetchHintPseudoExpansion(MI);
+ return;
case AArch64::ATOMIC_STORE_HINT_BroW:
case AArch64::ATOMIC_STORE_HINT_HroW:
case AArch64::ATOMIC_STORE_HINT_WroW:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2ddc7fe934430..71e3ad19cbe04 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -34099,11 +34099,15 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
}
// The !mem.cache_hint metadata is not supported by GISel and will be dropped.
- // TODO: Remove this and handle atomic store hints in GISel once supported.
+ // TODO: Remove this and handle atomic hints in GISel once supported.
if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
return true;
}
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst)) {
+ if (getMemCacheHintMetadata(*RMW))
+ return true;
+ }
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 09464116b2bfb..7568b6f530db8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -283,7 +283,7 @@ def : Pat<(relaxed_store<atomic_store_64>
(STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
//===----------------------------------
-// Atomic store with hint pseudos
+// Atomic operations with hint pseudos
//===----------------------------------
let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
@@ -441,6 +441,79 @@ let AddedComplexity = 15 in {
defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>;
}
+// Keep the hint and the LSE instruction together until assembly emission.
+// The opcode operand lets all operations and orderings share these pseudos.
+let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1,
+ mayStore = 1 in {
+ class BaseFetchHintPseudo<RegisterClass dsttype, RegisterClass srctype>
+ : Pseudo<(outs dsttype:$result),
+ (ins srctype:$data, GPR64sp:$addr, i32imm:$opcode,
+ i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ def ATOMIC_FETCH_HINT_W : BaseFetchHintPseudo<GPR32common, GPR32>;
+ def ATOMIC_FETCH_HINT_X : BaseFetchHintPseudo<GPR64common, GPR64>;
+}
+
+class atomic_hint_fetch<PatFrag Base, code Pred>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (Base node:$ptr, node:$val), Pred> {
+ // TODO: Change once mem.cache_hint supported in GISel
+ let GISelPredicateCode = [{ return false; }];
+}
+
+// Pass the LSE opcode as an immediate so all operations and orderings can use
+// the same two fetch pseudos.
+class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm,
+ "return CurDAG->getTargetConstant(AArch64::" # Inst #
+ ", SDLoc(N), MVT::i32);">;
+
+multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op,
+ RegisterClass RC, Instruction Pseudo> {
+ defvar Base = !cast<PatFrag>(Op);
+ defvar Opcode = AtomicFetchOpcode<Inst>;
+ def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>
+ GPR64sp:$addr, RC:$data),
+ (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 50))>;
+ def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>
+ GPR64sp:$addr, RC:$data),
+ (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 51))>;
+}
+
+multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op,
+ RegisterClass RC, Instruction Pseudo> {
+ defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic",
+ RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire",
+ RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release",
+ RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel",
+ RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst",
+ RC, Pseudo>;
+}
+
+multiclass AtomicFetchHintPatterns<string Inst, string Op> {
+ defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64,
+ ATOMIC_FETCH_HINT_X>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32,
+ ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32,
+ ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32,
+ ATOMIC_FETCH_HINT_W>;
+}
+
+// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a
+// complemented value. Hinted atomics use SelectionDAG, so separate SUB/AND
+// patterns are not needed here (unlike the GlobalISel patterns below).
+let Predicates = [HasLSE], AddedComplexity = 15 in {
+ defm : AtomicFetchHintPatterns<"LDADD", "atomic_load_add">;
+ defm : AtomicFetchHintPatterns<"LDSET", "atomic_load_or">;
+ defm : AtomicFetchHintPatterns<"LDEOR", "atomic_load_xor">;
+ defm : AtomicFetchHintPatterns<"LDCLR", "atomic_load_clr">;
+}
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index c989a5e40e091..b227f27633ce4 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -1,6 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+outline-atomics -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE
;
; STSHH: Keep, Relaxed
@@ -359,7 +362,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: strb w1, [x0]
; CHECK-NEXT: ret
store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6
@@ -369,7 +372,7 @@ define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_release_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6
@@ -386,6 +389,104 @@ define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
+define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; NOLSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
+; NOLSE: // %bb.0:
+; NOLSE-NEXT: .LBB33_1: // %atomicrmw.start
+; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT: ldxrb w8, [x0]
+; NOLSE-NEXT: add w9, w8, w1
+; NOLSE-NEXT: stxrb w10, w9, [x0]
+; NOLSE-NEXT: cbnz w10, .LBB33_1
+; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT: mov w0, w8
+; NOLSE-NEXT: ret
+;
+; LSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
+; LSE: // %bb.0:
+; LSE-NEXT: shuh
+; LSE-NEXT: ldaddb w1, w0, [x0]
+; LSE-NEXT: ret
+;
+; OUTLINE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov w0, w1
+; OUTLINE-NEXT: mov x1, x8
+; OUTLINE-NEXT: bl __aarch64_ldadd1_relax
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !12
+ ret i8 %old
+}
+
+define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
+; NOLSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
+; NOLSE: // %bb.0:
+; NOLSE-NEXT: .LBB34_1: // %atomicrmw.start
+; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT: ldaxrb w8, [x0]
+; NOLSE-NEXT: and w9, w8, w1
+; NOLSE-NEXT: stlxrb w10, w9, [x0]
+; NOLSE-NEXT: cbnz w10, .LBB34_1
+; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT: mov w0, w8
+; NOLSE-NEXT: ret
+;
+; LSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
+; LSE: // %bb.0:
+; LSE-NEXT: mvn w8, w1
+; LSE-NEXT: shuh
+; LSE-NEXT: ldclralb w8, w0, [x0]
+; LSE-NEXT: ret
+;
+; OUTLINE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mvn w0, w1
+; OUTLINE-NEXT: mov x1, x8
+; OUTLINE-NEXT: bl __aarch64_ldclr1_acq_rel
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !12
+ ret i8 %old
+}
+
+; Sequentially consistent 64-bit fetch-xor: retain acquire semantics when the
+; result is unused.
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind {
+; NOLSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
+; NOLSE: // %bb.0:
+; NOLSE-NEXT: .LBB35_1: // %atomicrmw.start
+; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT: ldaxr x8, [x0]
+; NOLSE-NEXT: eor x8, x8, x1
+; NOLSE-NEXT: stlxr w9, x8, [x0]
+; NOLSE-NEXT: cbnz w9, .LBB35_1
+; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT: ret
+;
+; LSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
+; LSE: // %bb.0:
+; LSE-NEXT: shuh
+; LSE-NEXT: ldeoral x1, x8, [x0]
+; LSE-NEXT: ret
+;
+; OUTLINE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov x0, x1
+; OUTLINE-NEXT: mov x1, x8
+; OUTLINE-NEXT: bl __aarch64_ldeor8_acq_rel
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !12
+ ret void
+}
+
;
; SHUH PH
;
@@ -420,6 +521,71 @@ define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
+define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind {
+; NOLSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
+; NOLSE: // %bb.0:
+; NOLSE-NEXT: .LBB39_1: // %atomicrmw.start
+; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT: ldaxr w8, [x0]
+; NOLSE-NEXT: sub w9, w8, w1
+; NOLSE-NEXT: stxr w10, w9, [x0]
+; NOLSE-NEXT: cbnz w10, .LBB39_1
+; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT: mov w0, w8
+; NOLSE-NEXT: ret
+;
+; LSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
+; LSE: // %bb.0:
+; LSE-NEXT: neg w8, w1
+; LSE-NEXT: shuh ph
+; LSE-NEXT: ldadda w8, w0, [x0]
+; LSE-NEXT: ret
+;
+; OUTLINE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: neg w0, w1
+; OUTLINE-NEXT: mov x1, x8
+; OUTLINE-NEXT: bl __aarch64_ldadd4_acq
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !13
+ ret i32 %old
+}
+
+define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind {
+; NOLSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
+; NOLSE: // %bb.0:
+; NOLSE-NEXT: .LBB40_1: // %atomicrmw.start
+; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
+; NOLSE-NEXT: ldxrh w8, [x0]
+; NOLSE-NEXT: orr w9, w8, w1
+; NOLSE-NEXT: stlxrh w10, w9, [x0]
+; NOLSE-NEXT: cbnz w10, .LBB40_1
+; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
+; NOLSE-NEXT: mov w0, w8
+; NOLSE-NEXT: ret
+;
+; LSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
+; LSE: // %bb.0:
+; LSE-NEXT: shuh ph
+; LSE-NEXT: ldsetlh w1, w0, [x0]
+; LSE-NEXT: ret
+;
+; OUTLINE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
+; OUTLINE: // %bb.0:
+; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; OUTLINE-NEXT: mov x8, x0
+; OUTLINE-NEXT: mov w0, w1
+; OUTLINE-NEXT: mov x1, x8
+; OUTLINE-NEXT: bl __aarch64_ldset2_rel
+; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; OUTLINE-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !13
+ ret i16 %old
+}
+
;
; Invalid Hint
;
@@ -445,3 +611,5 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
!9 = !{!"aarch64.mem_hint", i32 4}
!10 = !{i32 1, !11}
!11 = !{!"aarch64.mem_hint", i32 5}
+!12 = !{i32 0, !7}
+!13 = !{i32 0, !9}
>From 7abd7203682803a6b89a05ae7272b4b5add7ca4e Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 30 Sep 2026 13:28:54 +0000
Subject: [PATCH 05/11] Formatting
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 3293c263527d4..8ec2929eb8a81 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2067,8 +2067,7 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
}
static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
- Instruction *Atomic,
- unsigned PtrOperand) {
+ Instruction *Atomic, unsigned PtrOperand) {
CodeGen::CGBuilderTy &Builder = CGF.Builder;
LLVMContext &Ctx = CGF.CGM.getLLVMContext();
Expr::EvalResult Result;
@@ -2084,8 +2083,8 @@ static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
MDNode *HintNode = MDNode::get(
- Ctx, {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)),
- MemHint});
+ Ctx,
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), MemHint});
Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
}
>From 3dfb40c2134002494f1555edc415b0b2a06d33a8 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 7 Oct 2026 10:34:02 +0000
Subject: [PATCH 06/11] Address comments
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 5 +++-
clang/lib/Sema/SemaARM.cpp | 10 ++-----
clang/test/CodeGen/builtins-arm64.c | 13 ++++++++
clang/test/Sema/builtins-arm64.c | 8 ++---
.../include/llvm/Support/AArch64MemoryHints.h | 5 ----
.../lib/Target/AArch64/AArch64InstrAtomics.td | 30 +++++++------------
6 files changed, 34 insertions(+), 37 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 8ec2929eb8a81..02a30f8ec5b25 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2076,7 +2076,10 @@ static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
"Expected integer hint argument to atomic operation with hint.");
unsigned HintArg = Result.Val.getInt().getExtValue();
- if (toAArch64MemoryHint(HintArg) == AArch64MemoryHint::NONE)
+ auto MemoryHint = toAArch64MemoryHint(HintArg);
+ if (MemoryHint == AArch64MemoryHint::NONE ||
+ (isa<AtomicRMWInst>(Atomic) && MemoryHint != AArch64MemoryHint::SHUH &&
+ MemoryHint != AArch64MemoryHint::SHUH_PH))
return;
MDNode *MemHint = MDNode::get(
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 2f5a9f985a81d..a1e4df5300d2e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -440,13 +440,9 @@ bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) {
<< Hint->getType() << Hint->getSourceRange();
auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue());
- if (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH &&
- MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)
- return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_hint_type)
- << *HintAP << Hint->getSourceRange();
-
- if (MemoryHint == llvm::AArch64MemoryHint::NONE) {
+ if (MemoryHint == llvm::AArch64MemoryHint::NONE ||
+ (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH &&
+ MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)) {
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << Hint->getSourceRange();
return false;
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 33960feb6cdb2..ed83126ce5420 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -295,6 +295,19 @@ int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) {
return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4);
}
+// CHECK-LABEL: @atomic_fetch_with_invalid_hint(
+// CHECK: atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: atomicrmw sub ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: atomicrmw and ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_with_invalid_hint(int64_t *a, int64_t b) {
+ __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+ __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_STRM);
+ __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH);
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 5);
+}
+
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index ba0cf233df319..96774a02fca64 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -105,8 +105,8 @@ void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr,
__builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
__builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
// expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
- __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-error {{invalid hint type argument to atomic hint operation (0)}}
- __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-error {{invalid hint type argument to atomic hint operation (1)}}
- __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-error {{invalid hint type argument to atomic hint operation (2)}}
- __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-error {{invalid hint type argument to atomic hint operation (5)}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-warning {{unrecognised hint type argument to atomic hint operation (0)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-warning {{unrecognised hint type argument to atomic hint operation (1)}}
+ __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-warning {{unrecognised hint type argument to atomic hint operation (2)}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}}
}
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
index f47eb0c945d1b..aefd1c1076462 100644
--- a/llvm/include/llvm/Support/AArch64MemoryHints.h
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -19,11 +19,6 @@ enum class AArch64MemoryHint {
SHUH_PH = 5,
};
-template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
- return (Int)AArch64MemoryHint::STSHH_KEEP <= I &&
- I <= (Int)AArch64MemoryHint::SHUH_PH;
-}
-
template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
switch (I) {
case 0:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 7568b6f530db8..dce5d7d84e2d2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -464,8 +464,7 @@ class atomic_hint_fetch<PatFrag Base, code Pred>
// Pass the LSE opcode as an immediate so all operations and orderings can use
// the same two fetch pseudos.
class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm,
- "return CurDAG->getTargetConstant(AArch64::" # Inst #
- ", SDLoc(N), MVT::i32);">;
+ "return CurDAG->getTargetConstant(AArch64::" # Inst # ", SDLoc(N), MVT::i32);">;
multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op,
RegisterClass RC, Instruction Pseudo> {
@@ -481,27 +480,18 @@ multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op,
multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op,
RegisterClass RC, Instruction Pseudo> {
- defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic",
- RC, Pseudo>;
- defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire",
- RC, Pseudo>;
- defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release",
- RC, Pseudo>;
- defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel",
- RC, Pseudo>;
- defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst",
- RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", RC, Pseudo>;
}
multiclass AtomicFetchHintPatterns<string Inst, string Op> {
- defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64,
- ATOMIC_FETCH_HINT_X>;
- defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32,
- ATOMIC_FETCH_HINT_W>;
- defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32,
- ATOMIC_FETCH_HINT_W>;
- defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32,
- ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, ATOMIC_FETCH_HINT_X>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, ATOMIC_FETCH_HINT_W>;
}
// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a
>From 202bd732665282aeac503e250186921435dd5ae3 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 7 Oct 2026 11:04:25 +0000
Subject: [PATCH 07/11] Split tests
---
.../Atomics/aarch64-atomic-fetch-hint.ll | 394 ++++++++++++++++++
.../Atomics/aarch64-atomic-store-hint.ll | 176 +-------
2 files changed, 398 insertions(+), 172 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
new file mode 100644
index 0000000000000..af0fc646df4cd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
@@ -0,0 +1,394 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,GISEL
+
+;
+; SHUH, Relaxed
+;
+
+define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB0_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxrb w8, [x0]
+; CHECK-NEXT: add w9, w8, w1
+; CHECK-NEXT: stxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB0_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !0
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_add_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB1_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxrh w8, [x0]
+; CHECK-NEXT: add w9, w8, w1
+; CHECK-NEXT: stxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB1_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i16 %val monotonic, align 2, !mem.cache_hint !0
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_add_shuh_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB2_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxr w8, [x0]
+; CHECK-NEXT: add w9, w8, w1
+; CHECK-NEXT: stxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB2_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i32 %val monotonic, align 4, !mem.cache_hint !0
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_add_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+; SDAG-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
+; SDAG: // %bb.0:
+; SDAG-NEXT: mov x8, x0
+; SDAG-NEXT: .LBB3_1: // %atomicrmw.start
+; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: ldxr x0, [x8]
+; SDAG-NEXT: add x9, x0, x1
+; SDAG-NEXT: stxr w10, x9, [x8]
+; SDAG-NEXT: cbnz w10, .LBB3_1
+; SDAG-NEXT: // %bb.2: // %atomicrmw.end
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
+; GISEL: // %bb.0:
+; GISEL-NEXT: .LBB3_1: // %atomicrmw.start
+; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: ldxr x8, [x0]
+; GISEL-NEXT: add x9, x8, x1
+; GISEL-NEXT: stxr w10, x9, [x0]
+; GISEL-NEXT: cbnz w10, .LBB3_1
+; GISEL-NEXT: // %bb.2: // %atomicrmw.end
+; GISEL-NEXT: mov x0, x8
+; GISEL-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i64 %val monotonic, align 8, !mem.cache_hint !0
+ ret i64 %old
+}
+
+;
+; SHUH, AcquireRelease
+;
+
+define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrb w8, [x0]
+; CHECK-NEXT: and w9, w8, w1
+; CHECK-NEXT: stlxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB4_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !0
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_and_shuh_acq_rel_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrh w8, [x0]
+; CHECK-NEXT: and w9, w8, w1
+; CHECK-NEXT: stlxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB5_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i16 %val acq_rel, align 2, !mem.cache_hint !0
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_and_shuh_acq_rel_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB6_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr w8, [x0]
+; CHECK-NEXT: and w9, w8, w1
+; CHECK-NEXT: stlxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB6_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i32 %val acq_rel, align 4, !mem.cache_hint !0
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_and_shuh_acq_rel_i64(ptr %ptr, i64 %val) nounwind {
+; SDAG-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
+; SDAG: // %bb.0:
+; SDAG-NEXT: mov x8, x0
+; SDAG-NEXT: .LBB7_1: // %atomicrmw.start
+; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: ldaxr x0, [x8]
+; SDAG-NEXT: and x9, x0, x1
+; SDAG-NEXT: stlxr w10, x9, [x8]
+; SDAG-NEXT: cbnz w10, .LBB7_1
+; SDAG-NEXT: // %bb.2: // %atomicrmw.end
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
+; GISEL: // %bb.0:
+; GISEL-NEXT: .LBB7_1: // %atomicrmw.start
+; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: ldaxr x8, [x0]
+; GISEL-NEXT: and x9, x8, x1
+; GISEL-NEXT: stlxr w10, x9, [x0]
+; GISEL-NEXT: cbnz w10, .LBB7_1
+; GISEL-NEXT: // %bb.2: // %atomicrmw.end
+; GISEL-NEXT: mov x0, x8
+; GISEL-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i64 %val acq_rel, align 8, !mem.cache_hint !0
+ ret i64 %old
+}
+
+;
+; SHUH, SequentiallyConsistent
+;
+
+; Sequentially consistent fetch-xor: retain acquire semantics when the
+; result is unused.
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB8_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrb w8, [x0]
+; CHECK-NEXT: eor w8, w8, w1
+; CHECK-NEXT: stlxrb w9, w8, [x0]
+; CHECK-NEXT: cbnz w9, .LBB8_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i8 %val seq_cst, align 1, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB9_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrh w8, [x0]
+; CHECK-NEXT: eor w8, w8, w1
+; CHECK-NEXT: stlxrh w9, w8, [x0]
+; CHECK-NEXT: cbnz w9, .LBB9_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i16 %val seq_cst, align 2, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB10_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr w8, [x0]
+; CHECK-NEXT: eor w8, w8, w1
+; CHECK-NEXT: stlxr w9, w8, [x0]
+; CHECK-NEXT: cbnz w9, .LBB10_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i32 %val seq_cst, align 4, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB11_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr x8, [x0]
+; CHECK-NEXT: eor x8, x8, x1
+; CHECK-NEXT: stlxr w9, x8, [x0]
+; CHECK-NEXT: cbnz w9, .LBB11_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; SHUH PH, Acquire
+;
+
+define i8 @test_atomic_fetch_sub_shuh_ph_acquire_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB12_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrb w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: stxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB12_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i8 %val acquire, align 1, !mem.cache_hint !2
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_sub_shuh_ph_acquire_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB13_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxrh w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: stxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB13_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i16 %val acquire, align 2, !mem.cache_hint !2
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB14_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldaxr w8, [x0]
+; CHECK-NEXT: sub w9, w8, w1
+; CHECK-NEXT: stxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB14_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_sub_shuh_ph_acquire_i64(ptr %ptr, i64 %val) nounwind {
+; SDAG-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
+; SDAG: // %bb.0:
+; SDAG-NEXT: mov x8, x0
+; SDAG-NEXT: .LBB15_1: // %atomicrmw.start
+; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: ldaxr x0, [x8]
+; SDAG-NEXT: sub x9, x0, x1
+; SDAG-NEXT: stxr w10, x9, [x8]
+; SDAG-NEXT: cbnz w10, .LBB15_1
+; SDAG-NEXT: // %bb.2: // %atomicrmw.end
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
+; GISEL: // %bb.0:
+; GISEL-NEXT: .LBB15_1: // %atomicrmw.start
+; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: ldaxr x8, [x0]
+; GISEL-NEXT: sub x9, x8, x1
+; GISEL-NEXT: stxr w10, x9, [x0]
+; GISEL-NEXT: cbnz w10, .LBB15_1
+; GISEL-NEXT: // %bb.2: // %atomicrmw.end
+; GISEL-NEXT: mov x0, x8
+; GISEL-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i64 %val acquire, align 8, !mem.cache_hint !2
+ ret i64 %old
+}
+
+;
+; SHUH PH, Release
+;
+
+define i8 @test_atomic_fetch_or_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB16_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxrb w8, [x0]
+; CHECK-NEXT: orr w9, w8, w1
+; CHECK-NEXT: stlxrb w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB16_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i8 %val release, align 1, !mem.cache_hint !2
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB17_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxrh w8, [x0]
+; CHECK-NEXT: orr w9, w8, w1
+; CHECK-NEXT: stlxrh w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB17_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !2
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_or_shuh_ph_release_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: .LBB18_1: // %atomicrmw.start
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldxr w8, [x0]
+; CHECK-NEXT: orr w9, w8, w1
+; CHECK-NEXT: stlxr w10, w9, [x0]
+; CHECK-NEXT: cbnz w10, .LBB18_1
+; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i32 %val release, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_or_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind {
+; SDAG-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
+; SDAG: // %bb.0:
+; SDAG-NEXT: mov x8, x0
+; SDAG-NEXT: .LBB19_1: // %atomicrmw.start
+; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
+; SDAG-NEXT: ldxr x0, [x8]
+; SDAG-NEXT: orr x9, x0, x1
+; SDAG-NEXT: stlxr w10, x9, [x8]
+; SDAG-NEXT: cbnz w10, .LBB19_1
+; SDAG-NEXT: // %bb.2: // %atomicrmw.end
+; SDAG-NEXT: ret
+;
+; GISEL-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
+; GISEL: // %bb.0:
+; GISEL-NEXT: .LBB19_1: // %atomicrmw.start
+; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
+; GISEL-NEXT: ldxr x8, [x0]
+; GISEL-NEXT: orr x9, x8, x1
+; GISEL-NEXT: stlxr w10, x9, [x0]
+; GISEL-NEXT: cbnz w10, .LBB19_1
+; GISEL-NEXT: // %bb.2: // %atomicrmw.end
+; GISEL-NEXT: mov x0, x8
+; GISEL-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i64 %val release, align 8, !mem.cache_hint !2
+ ret i64 %old
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"aarch64.mem_hint", i32 3}
+!2 = !{i32 0, !3}
+!3 = !{!"aarch64.mem_hint", i32 4}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index 6f36531cb8dad..06dc124c79088 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -1,9 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,LSE
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+outline-atomics -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,OUTLINE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; STSHH: Keep, Relaxed
@@ -460,7 +457,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: strb w1, [x0]
; CHECK-NEXT: ret
store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6
@@ -534,7 +531,7 @@ define void @test_atomic_store_shuh_release_i32(ptr %ptr, i32 %val) nounwind {
define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_release_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6
@@ -585,104 +582,6 @@ define void @test_atomic_store_shuh_seqcst_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
-define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
-; NOLSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
-; NOLSE: // %bb.0:
-; NOLSE-NEXT: .LBB51_1: // %atomicrmw.start
-; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT: ldxrb w8, [x0]
-; NOLSE-NEXT: add w9, w8, w1
-; NOLSE-NEXT: stxrb w10, w9, [x0]
-; NOLSE-NEXT: cbnz w10, .LBB51_1
-; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT: mov w0, w8
-; NOLSE-NEXT: ret
-;
-; LSE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
-; LSE: // %bb.0:
-; LSE-NEXT: shuh
-; LSE-NEXT: ldaddb w1, w0, [x0]
-; LSE-NEXT: ret
-;
-; OUTLINE-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
-; OUTLINE: // %bb.0:
-; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
-; OUTLINE-NEXT: mov x8, x0
-; OUTLINE-NEXT: mov w0, w1
-; OUTLINE-NEXT: mov x1, x8
-; OUTLINE-NEXT: bl __aarch64_ldadd1_relax
-; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
-; OUTLINE-NEXT: ret
- %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !12
- ret i8 %old
-}
-
-define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
-; NOLSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
-; NOLSE: // %bb.0:
-; NOLSE-NEXT: .LBB52_1: // %atomicrmw.start
-; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT: ldaxrb w8, [x0]
-; NOLSE-NEXT: and w9, w8, w1
-; NOLSE-NEXT: stlxrb w10, w9, [x0]
-; NOLSE-NEXT: cbnz w10, .LBB52_1
-; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT: mov w0, w8
-; NOLSE-NEXT: ret
-;
-; LSE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
-; LSE: // %bb.0:
-; LSE-NEXT: mvn w8, w1
-; LSE-NEXT: shuh
-; LSE-NEXT: ldclralb w8, w0, [x0]
-; LSE-NEXT: ret
-;
-; OUTLINE-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
-; OUTLINE: // %bb.0:
-; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
-; OUTLINE-NEXT: mov x8, x0
-; OUTLINE-NEXT: mvn w0, w1
-; OUTLINE-NEXT: mov x1, x8
-; OUTLINE-NEXT: bl __aarch64_ldclr1_acq_rel
-; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
-; OUTLINE-NEXT: ret
- %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !12
- ret i8 %old
-}
-
-; Sequentially consistent 64-bit fetch-xor: retain acquire semantics when the
-; result is unused.
-define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind {
-; NOLSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
-; NOLSE: // %bb.0:
-; NOLSE-NEXT: .LBB53_1: // %atomicrmw.start
-; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT: ldaxr x8, [x0]
-; NOLSE-NEXT: eor x8, x8, x1
-; NOLSE-NEXT: stlxr w9, x8, [x0]
-; NOLSE-NEXT: cbnz w9, .LBB53_1
-; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT: ret
-;
-; LSE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
-; LSE: // %bb.0:
-; LSE-NEXT: shuh
-; LSE-NEXT: ldeoral x1, x8, [x0]
-; LSE-NEXT: ret
-;
-; OUTLINE-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
-; OUTLINE: // %bb.0:
-; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
-; OUTLINE-NEXT: mov x8, x0
-; OUTLINE-NEXT: mov x0, x1
-; OUTLINE-NEXT: mov x1, x8
-; OUTLINE-NEXT: bl __aarch64_ldeor8_acq_rel
-; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
-; OUTLINE-NEXT: ret
- %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !12
- ret void
-}
-
;
; SHUH PH, Relaxed
;
@@ -815,71 +714,6 @@ define void @test_atomic_store_shuh_ph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
ret void
}
-define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind {
-; NOLSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
-; NOLSE: // %bb.0:
-; NOLSE-NEXT: .LBB66_1: // %atomicrmw.start
-; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT: ldaxr w8, [x0]
-; NOLSE-NEXT: sub w9, w8, w1
-; NOLSE-NEXT: stxr w10, w9, [x0]
-; NOLSE-NEXT: cbnz w10, .LBB66_1
-; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT: mov w0, w8
-; NOLSE-NEXT: ret
-;
-; LSE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
-; LSE: // %bb.0:
-; LSE-NEXT: neg w8, w1
-; LSE-NEXT: shuh ph
-; LSE-NEXT: ldadda w8, w0, [x0]
-; LSE-NEXT: ret
-;
-; OUTLINE-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
-; OUTLINE: // %bb.0:
-; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
-; OUTLINE-NEXT: mov x8, x0
-; OUTLINE-NEXT: neg w0, w1
-; OUTLINE-NEXT: mov x1, x8
-; OUTLINE-NEXT: bl __aarch64_ldadd4_acq
-; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
-; OUTLINE-NEXT: ret
- %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !13
- ret i32 %old
-}
-
-define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind {
-; NOLSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
-; NOLSE: // %bb.0:
-; NOLSE-NEXT: .LBB67_1: // %atomicrmw.start
-; NOLSE-NEXT: // =>This Inner Loop Header: Depth=1
-; NOLSE-NEXT: ldxrh w8, [x0]
-; NOLSE-NEXT: orr w9, w8, w1
-; NOLSE-NEXT: stlxrh w10, w9, [x0]
-; NOLSE-NEXT: cbnz w10, .LBB67_1
-; NOLSE-NEXT: // %bb.2: // %atomicrmw.end
-; NOLSE-NEXT: mov w0, w8
-; NOLSE-NEXT: ret
-;
-; LSE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
-; LSE: // %bb.0:
-; LSE-NEXT: shuh ph
-; LSE-NEXT: ldsetlh w1, w0, [x0]
-; LSE-NEXT: ret
-;
-; OUTLINE-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
-; OUTLINE: // %bb.0:
-; OUTLINE-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
-; OUTLINE-NEXT: mov x8, x0
-; OUTLINE-NEXT: mov w0, w1
-; OUTLINE-NEXT: mov x1, x8
-; OUTLINE-NEXT: bl __aarch64_ldset2_rel
-; OUTLINE-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
-; OUTLINE-NEXT: ret
- %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !13
- ret i16 %old
-}
-
;
; Invalid Hint
;
@@ -905,5 +739,3 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
!9 = !{!"aarch64.mem_hint", i32 4}
!10 = !{i32 1, !11}
!11 = !{!"aarch64.mem_hint", i32 5}
-!12 = !{i32 0, !7}
-!13 = !{i32 0, !9}
>From 0ab348d80aaa3edc67279ae620e494e3891a5f25 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Wed, 7 Oct 2026 15:51:47 +0000
Subject: [PATCH 08/11] Fix test
---
clang/test/CodeGen/builtins-arm64.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index b203c100b815a..ed83126ce5420 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -241,7 +241,6 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
- __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
// CHECK-NOT: !mem.cache_hint
// CHECK-NEXT: ret void
>From 0904e30c51207b24e7a5fd3d39847dd0a704f01a Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Fri, 9 Oct 2026 10:25:12 +0000
Subject: [PATCH 09/11] Address comments
---
.../AArch64/atomic-store-hint-template.cpp | 23 ++++++++-----------
.../Target/AArch64/AArch64ISelLowering.cpp | 3 +--
.../lib/Target/AArch64/AArch64InstrAtomics.td | 2 +-
3 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
index 17e99be63adcd..0f64aacd6d969 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -57,6 +57,11 @@ void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) {
template <unsigned Order>
int test_atomic_fetch_hint_template_order(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]]
+// CHECK-NEXT: ret i32 [[TMP0]]
return __arm_atomic_fetch_add_with_hint(ptr, val, Order, 3);
}
@@ -64,24 +69,16 @@ template int test_atomic_fetch_hint_template_order<__ATOMIC_ACQUIRE>(int *, int)
template <unsigned Hint>
int test_atomic_fetch_hint_template_hint(int *ptr, int val) {
- return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
-}
-
-template int test_atomic_fetch_hint_template_hint<4>(int *, int);
-
-// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii(
-// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
-// CHECK-NEXT: [[ENTRY:.*:]]
-// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]]
-// CHECK-NEXT: ret i32 [[TMP0]]
-//
-//
// CHECK-LABEL: define weak_odr noundef i32 @_Z36test_atomic_fetch_hint_template_hintILj4EEiPii(
// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
// CHECK-NEXT: [[ENTRY:.*:]]
// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw xor ptr [[PTR]], i32 [[VAL]] seq_cst, align 4, !mem.cache_hint [[META10:![0-9]+]]
// CHECK-NEXT: ret i32 [[TMP0]]
-//
+ return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
+}
+
+template int test_atomic_fetch_hint_template_hint<4>(int *, int);
+
//.
// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e1066a0cfd145..34314253004ff 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -34107,10 +34107,9 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
return true;
}
- if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst)) {
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst))
if (getMemCacheHintMetadata(*RMW))
return true;
- }
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index dce5d7d84e2d2..60c2e796a6c13 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -497,7 +497,7 @@ multiclass AtomicFetchHintPatterns<string Inst, string Op> {
// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a
// complemented value. Hinted atomics use SelectionDAG, so separate SUB/AND
// patterns are not needed here (unlike the GlobalISel patterns below).
-let Predicates = [HasLSE], AddedComplexity = 15 in {
+let Predicates = [HasLSE], AddedComplexity = 1 in {
defm : AtomicFetchHintPatterns<"LDADD", "atomic_load_add">;
defm : AtomicFetchHintPatterns<"LDSET", "atomic_load_or">;
defm : AtomicFetchHintPatterns<"LDEOR", "atomic_load_xor">;
>From 1ed38b6b60586bb57577c28aee90dfd1eefd5e75 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Mon, 12 Oct 2026 17:02:48 +0000
Subject: [PATCH 10/11] Address comments
---
...-template.cpp => atomic-hint-template.cpp} | 0
.../lib/Target/AArch64/AArch64InstrAtomics.td | 5 +-
.../Atomics/aarch64-atomic-fetch-hint.ll | 283 ++++--------------
3 files changed, 66 insertions(+), 222 deletions(-)
rename clang/test/CodeGen/AArch64/{atomic-store-hint-template.cpp => atomic-hint-template.cpp} (100%)
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-hint-template.cpp
similarity index 100%
rename from clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
rename to clang/test/CodeGen/AArch64/atomic-hint-template.cpp
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index c9f81e2d5232a..6564971876d9e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -453,10 +453,7 @@ let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1,
class atomic_hint_fetch<PatFrag Base, code Pred>
: PatFrag<(ops node:$ptr, node:$val),
- (Base node:$ptr, node:$val), Pred> {
- // TODO: Change once mem.cache_hint supported in GISel
- let GISelPredicateCode = [{ return false; }];
-}
+ (Base node:$ptr, node:$val), Pred>;
// Pass the LSE opcode as an immediate so all operations and orderings can use
// the same two fetch pseudos.
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
index af0fc646df4cd..93ea92d5de46b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,SDAG
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,GISEL
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
;
; SHUH, Relaxed
@@ -9,14 +9,8 @@
define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB0_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxrb w8, [x0]
-; CHECK-NEXT: add w9, w8, w1
-; CHECK-NEXT: stxrb w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB0_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldaddb w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !0
ret i8 %old
@@ -25,14 +19,8 @@ define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
define i16 @test_atomic_fetch_add_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB1_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxrh w8, [x0]
-; CHECK-NEXT: add w9, w8, w1
-; CHECK-NEXT: stxrh w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB1_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldaddh w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw add ptr %ptr, i16 %val monotonic, align 2, !mem.cache_hint !0
ret i16 %old
@@ -41,43 +29,20 @@ define i16 @test_atomic_fetch_add_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind
define i32 @test_atomic_fetch_add_shuh_relaxed_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB2_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxr w8, [x0]
-; CHECK-NEXT: add w9, w8, w1
-; CHECK-NEXT: stxr w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB2_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldadd w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw add ptr %ptr, i32 %val monotonic, align 4, !mem.cache_hint !0
ret i32 %old
}
define i64 @test_atomic_fetch_add_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind {
-; SDAG-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
-; SDAG: // %bb.0:
-; SDAG-NEXT: mov x8, x0
-; SDAG-NEXT: .LBB3_1: // %atomicrmw.start
-; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: ldxr x0, [x8]
-; SDAG-NEXT: add x9, x0, x1
-; SDAG-NEXT: stxr w10, x9, [x8]
-; SDAG-NEXT: cbnz w10, .LBB3_1
-; SDAG-NEXT: // %bb.2: // %atomicrmw.end
-; SDAG-NEXT: ret
-;
-; GISEL-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
-; GISEL: // %bb.0:
-; GISEL-NEXT: .LBB3_1: // %atomicrmw.start
-; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: ldxr x8, [x0]
-; GISEL-NEXT: add x9, x8, x1
-; GISEL-NEXT: stxr w10, x9, [x0]
-; GISEL-NEXT: cbnz w10, .LBB3_1
-; GISEL-NEXT: // %bb.2: // %atomicrmw.end
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: ret
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldadd x1, x0, [x0]
+; CHECK-NEXT: ret
+; CHECK-
%old = atomicrmw add ptr %ptr, i64 %val monotonic, align 8, !mem.cache_hint !0
ret i64 %old
}
@@ -89,14 +54,9 @@ define i64 @test_atomic_fetch_add_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind
define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB4_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrb w8, [x0]
-; CHECK-NEXT: and w9, w8, w1
-; CHECK-NEXT: stlxrb w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB4_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclralb w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !0
ret i8 %old
@@ -105,14 +65,9 @@ define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
define i16 @test_atomic_fetch_and_shuh_acq_rel_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB5_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrh w8, [x0]
-; CHECK-NEXT: and w9, w8, w1
-; CHECK-NEXT: stlxrh w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB5_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclralh w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw and ptr %ptr, i16 %val acq_rel, align 2, !mem.cache_hint !0
ret i16 %old
@@ -121,43 +76,22 @@ define i16 @test_atomic_fetch_and_shuh_acq_rel_i16(ptr %ptr, i16 %val) nounwind
define i32 @test_atomic_fetch_and_shuh_acq_rel_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB6_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxr w8, [x0]
-; CHECK-NEXT: and w9, w8, w1
-; CHECK-NEXT: stlxr w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB6_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclral w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw and ptr %ptr, i32 %val acq_rel, align 4, !mem.cache_hint !0
ret i32 %old
}
define i64 @test_atomic_fetch_and_shuh_acq_rel_i64(ptr %ptr, i64 %val) nounwind {
-; SDAG-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
-; SDAG: // %bb.0:
-; SDAG-NEXT: mov x8, x0
-; SDAG-NEXT: .LBB7_1: // %atomicrmw.start
-; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: ldaxr x0, [x8]
-; SDAG-NEXT: and x9, x0, x1
-; SDAG-NEXT: stlxr w10, x9, [x8]
-; SDAG-NEXT: cbnz w10, .LBB7_1
-; SDAG-NEXT: // %bb.2: // %atomicrmw.end
-; SDAG-NEXT: ret
-;
-; GISEL-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
-; GISEL: // %bb.0:
-; GISEL-NEXT: .LBB7_1: // %atomicrmw.start
-; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: ldaxr x8, [x0]
-; GISEL-NEXT: and x9, x8, x1
-; GISEL-NEXT: stlxr w10, x9, [x0]
-; GISEL-NEXT: cbnz w10, .LBB7_1
-; GISEL-NEXT: // %bb.2: // %atomicrmw.end
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: ret
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn x8, x1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclral x8, x0, [x0]
+; CHECK-NEXT: ret
+
%old = atomicrmw and ptr %ptr, i64 %val acq_rel, align 8, !mem.cache_hint !0
ret i64 %old
}
@@ -171,13 +105,8 @@ define i64 @test_atomic_fetch_and_shuh_acq_rel_i64(ptr %ptr, i64 %val) nounwind
define void @test_atomic_fetch_xor_shuh_seqcst_dead_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB8_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrb w8, [x0]
-; CHECK-NEXT: eor w8, w8, w1
-; CHECK-NEXT: stlxrb w9, w8, [x0]
-; CHECK-NEXT: cbnz w9, .LBB8_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoralb w1, w8, [x0]
; CHECK-NEXT: ret
%old = atomicrmw xor ptr %ptr, i8 %val seq_cst, align 1, !mem.cache_hint !0
ret void
@@ -186,13 +115,8 @@ define void @test_atomic_fetch_xor_shuh_seqcst_dead_i8(ptr %ptr, i8 %val) nounwi
define void @test_atomic_fetch_xor_shuh_seqcst_dead_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB9_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrh w8, [x0]
-; CHECK-NEXT: eor w8, w8, w1
-; CHECK-NEXT: stlxrh w9, w8, [x0]
-; CHECK-NEXT: cbnz w9, .LBB9_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoralh w1, w8, [x0]
; CHECK-NEXT: ret
%old = atomicrmw xor ptr %ptr, i16 %val seq_cst, align 2, !mem.cache_hint !0
ret void
@@ -201,13 +125,8 @@ define void @test_atomic_fetch_xor_shuh_seqcst_dead_i16(ptr %ptr, i16 %val) noun
define void @test_atomic_fetch_xor_shuh_seqcst_dead_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB10_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxr w8, [x0]
-; CHECK-NEXT: eor w8, w8, w1
-; CHECK-NEXT: stlxr w9, w8, [x0]
-; CHECK-NEXT: cbnz w9, .LBB10_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoral w1, w8, [x0]
; CHECK-NEXT: ret
%old = atomicrmw xor ptr %ptr, i32 %val seq_cst, align 4, !mem.cache_hint !0
ret void
@@ -216,13 +135,8 @@ define void @test_atomic_fetch_xor_shuh_seqcst_dead_i32(ptr %ptr, i32 %val) noun
define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB11_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxr x8, [x0]
-; CHECK-NEXT: eor x8, x8, x1
-; CHECK-NEXT: stlxr w9, x8, [x0]
-; CHECK-NEXT: cbnz w9, .LBB11_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoral x1, x8, [x0]
; CHECK-NEXT: ret
%old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !0
ret void
@@ -235,14 +149,9 @@ define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) noun
define i8 @test_atomic_fetch_sub_shuh_ph_acquire_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB12_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrb w8, [x0]
-; CHECK-NEXT: sub w9, w8, w1
-; CHECK-NEXT: stxrb w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB12_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldaddab w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw sub ptr %ptr, i8 %val acquire, align 1, !mem.cache_hint !2
ret i8 %old
@@ -251,14 +160,9 @@ define i8 @test_atomic_fetch_sub_shuh_ph_acquire_i8(ptr %ptr, i8 %val) nounwind
define i16 @test_atomic_fetch_sub_shuh_ph_acquire_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB13_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxrh w8, [x0]
-; CHECK-NEXT: sub w9, w8, w1
-; CHECK-NEXT: stxrh w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB13_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldaddah w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw sub ptr %ptr, i16 %val acquire, align 2, !mem.cache_hint !2
ret i16 %old
@@ -267,43 +171,22 @@ define i16 @test_atomic_fetch_sub_shuh_ph_acquire_i16(ptr %ptr, i16 %val) nounwi
define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB14_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldaxr w8, [x0]
-; CHECK-NEXT: sub w9, w8, w1
-; CHECK-NEXT: stxr w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB14_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldadda w8, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !2
ret i32 %old
}
define i64 @test_atomic_fetch_sub_shuh_ph_acquire_i64(ptr %ptr, i64 %val) nounwind {
-; SDAG-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
-; SDAG: // %bb.0:
-; SDAG-NEXT: mov x8, x0
-; SDAG-NEXT: .LBB15_1: // %atomicrmw.start
-; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: ldaxr x0, [x8]
-; SDAG-NEXT: sub x9, x0, x1
-; SDAG-NEXT: stxr w10, x9, [x8]
-; SDAG-NEXT: cbnz w10, .LBB15_1
-; SDAG-NEXT: // %bb.2: // %atomicrmw.end
-; SDAG-NEXT: ret
-;
-; GISEL-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
-; GISEL: // %bb.0:
-; GISEL-NEXT: .LBB15_1: // %atomicrmw.start
-; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: ldaxr x8, [x0]
-; GISEL-NEXT: sub x9, x8, x1
-; GISEL-NEXT: stxr w10, x9, [x0]
-; GISEL-NEXT: cbnz w10, .LBB15_1
-; GISEL-NEXT: // %bb.2: // %atomicrmw.end
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: ret
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg x8, x1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldadda x8, x0, [x0]
+; CHECK-NEXT: ret
+
%old = atomicrmw sub ptr %ptr, i64 %val acquire, align 8, !mem.cache_hint !2
ret i64 %old
}
@@ -315,14 +198,8 @@ define i64 @test_atomic_fetch_sub_shuh_ph_acquire_i64(ptr %ptr, i64 %val) nounwi
define i8 @test_atomic_fetch_or_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB16_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxrb w8, [x0]
-; CHECK-NEXT: orr w9, w8, w1
-; CHECK-NEXT: stlxrb w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB16_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetlb w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw or ptr %ptr, i8 %val release, align 1, !mem.cache_hint !2
ret i8 %old
@@ -331,14 +208,8 @@ define i8 @test_atomic_fetch_or_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind {
define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB17_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxrh w8, [x0]
-; CHECK-NEXT: orr w9, w8, w1
-; CHECK-NEXT: stlxrh w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB17_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetlh w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !2
ret i16 %old
@@ -347,43 +218,19 @@ define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwin
define i32 @test_atomic_fetch_or_shuh_ph_release_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: .LBB18_1: // %atomicrmw.start
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldxr w8, [x0]
-; CHECK-NEXT: orr w9, w8, w1
-; CHECK-NEXT: stlxr w10, w9, [x0]
-; CHECK-NEXT: cbnz w10, .LBB18_1
-; CHECK-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-NEXT: mov w0, w8
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetl w1, w0, [x0]
; CHECK-NEXT: ret
%old = atomicrmw or ptr %ptr, i32 %val release, align 4, !mem.cache_hint !2
ret i32 %old
}
define i64 @test_atomic_fetch_or_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind {
-; SDAG-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
-; SDAG: // %bb.0:
-; SDAG-NEXT: mov x8, x0
-; SDAG-NEXT: .LBB19_1: // %atomicrmw.start
-; SDAG-NEXT: // =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: ldxr x0, [x8]
-; SDAG-NEXT: orr x9, x0, x1
-; SDAG-NEXT: stlxr w10, x9, [x8]
-; SDAG-NEXT: cbnz w10, .LBB19_1
-; SDAG-NEXT: // %bb.2: // %atomicrmw.end
-; SDAG-NEXT: ret
-;
-; GISEL-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
-; GISEL: // %bb.0:
-; GISEL-NEXT: .LBB19_1: // %atomicrmw.start
-; GISEL-NEXT: // =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: ldxr x8, [x0]
-; GISEL-NEXT: orr x9, x8, x1
-; GISEL-NEXT: stlxr w10, x9, [x0]
-; GISEL-NEXT: cbnz w10, .LBB19_1
-; GISEL-NEXT: // %bb.2: // %atomicrmw.end
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: ret
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetl x1, x0, [x0]
+; CHECK-NEXT: ret
%old = atomicrmw or ptr %ptr, i64 %val release, align 8, !mem.cache_hint !2
ret i64 %old
}
>From 4b5576a106e9fe1ba3cf6f9a0e4bc9c6041588aa Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac at arm.com>
Date: Tue, 13 Oct 2026 11:23:52 +0100
Subject: [PATCH 11/11] Update
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
Co-authored-by: Kerry McLaughlin <kerry.mclaughlin at arm.com>
---
.../test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
index 93ea92d5de46b..aae55ab0c25cb 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
-; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; SHUH, Relaxed
More information about the cfe-commits
mailing list