[clang] 336ad6e - [AArch64] Implement the atomic fetch with hint intrinsic (#227711)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 14 03:28:10 PDT 2026
Author: Lukacma
Date: 2026-10-14T10:28:04Z
New Revision: 336ad6e9bb9e0d140831fa1e4bc36e03ec88a808
URL: https://github.com/llvm/llvm-project/commit/336ad6e9bb9e0d140831fa1e4bc36e03ec88a808
DIFF: https://github.com/llvm/llvm-project/commit/336ad6e9bb9e0d140831fa1e4bc36e03ec88a808.diff
LOG: [AArch64] Implement the atomic fetch with hint intrinsic (#227711)
This patch extends the work started in #198316 and introduces fetch hint
intrinsic described in
[ACLE](https://arm-software.github.io/acle/main/acle.html#atomic-fetch-with-hints-intrinsics)
:
```
type __arm_atomic_fetch_add_with_hint(type *ptr, type data, int memory_order, int hint);
type __arm_atomic_fetch_sub_with_hint(type *ptr, type data, int memory_order, int hint);
type __arm_atomic_fetch_and_with_hint(type *ptr, type data, int memory_order, int hint);
type __arm_atomic_fetch_xor_with_hint(type *ptr, type data, int memory_order, int hint);
type __arm_atomic_fetch_or_with_hint(type *ptr, type data, int memory_order, int hint);
```
These intrinsics support scalar integral types of 8, 16, 32, and 64 bits
and only work with HINT_SHUH, HINT_SHUH_PH .
Added:
clang/test/CodeGen/AArch64/atomic-hint-template.cpp
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
Modified:
clang/include/clang/Basic/BuiltinsAArch64.td
clang/include/clang/Basic/DiagnosticSemaKinds.td
clang/include/clang/Sema/SemaARM.h
clang/lib/CodeGen/TargetBuiltins/ARM.cpp
clang/lib/Headers/arm_acle.h
clang/lib/Sema/SemaARM.cpp
clang/test/CodeGen/arm_acle.c
clang/test/CodeGen/builtins-arm64.c
clang/test/Sema/builtins-arm64.c
llvm/include/llvm/Support/AArch64MemoryHints.h
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64InstrAtomics.td
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
Removed:
clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
################################################################################
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 729045d8eb5176..84ad46119682a6 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -179,6 +179,11 @@ let Attributes = [NoThrow], Features = "ls64" in {
let Attributes = [NoThrow, CustomTypeChecking] in {
def atomic_store_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_add_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_sub_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_and_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_xor_with_hint : AArch64Builtin<"void(...)">;
+ def atomic_fetch_or_with_hint : AArch64Builtin<"void(...)">;
}
// Armv9.3-A Guarded Control Stack
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 1054ab293a96b3..162106c84d120d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9636,7 +9636,7 @@ def err_atomic_builtin_must_be_pointer_intptr : Error<
def err_atomic_hint_builtin_must_be_pointer : Error<
"address argument to atomic hint builtin must be a pointer to a scalar "
- "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+ "%select{integral|integral or floating-point}1 type of 8, 16, 32, or 64 bits (%0 invalid)">;
def err_atomic_hint_has_invalid_hint_type : Error<
"invalid hint type argument to atomic hint operation (%0)">;
@@ -9709,7 +9709,7 @@ def err_atomic_op_needs_atomic_fp
: Error<"address argument to atomic operation must be a pointer to "
"%select{|atomic }0floating point type (%1 invalid)">;
def err_atomic_op_hint_data_size : Error<
- "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
+ "address argument to atomic %select{fetch|store}0 with hint must be of size 8, 16, 32 or 64 bits">;
def err_atomic_hint_has_invalid_memory_order : Error<
"invalid memory order argument to atomic hint operation (%0 invalid)">;
def warn_atomic_hint_has_invalid_hint_type : Warning<
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index b0a01c40ffecef..76a06f4f4221a5 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -70,7 +70,7 @@ class SemaARM : public SemaBase {
bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum,
unsigned ExpectedFieldNum, bool AllowName);
bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
- bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall);
bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index fcd765aa64d0e8..4a7bdba6d917ee 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2067,6 +2067,31 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
Ops);
}
+static void AttachAtomicHintMetadata(CodeGenFunction &CGF, const CallExpr *E,
+ Instruction *Atomic, unsigned PtrOperand) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ LLVMContext &Ctx = CGF.CGM.getLLVMContext();
+ Expr::EvalResult Result;
+ if (!E->getArg(3)->EvaluateAsInt(Result, CGF.getContext()))
+ llvm_unreachable(
+ "Expected integer hint argument to atomic operation with hint.");
+ unsigned HintArg = Result.Val.getInt().getExtValue();
+
+ auto MemoryHint = toAArch64MemoryHint(HintArg);
+ if (MemoryHint == AArch64MemoryHint::NONE ||
+ (isa<AtomicRMWInst>(Atomic) && MemoryHint != AArch64MemoryHint::SHUH &&
+ MemoryHint != AArch64MemoryHint::SHUH_PH))
+ return;
+
+ MDNode *MemHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
+ MDNode *HintNode = MDNode::get(
+ Ctx,
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(PtrOperand)), MemHint});
+ Atomic->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
+}
+
static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned BuiltinID,
const CallExpr *E) {
@@ -2106,26 +2131,53 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
break;
}
Store->setAtomic(Ordering);
+ AttachAtomicHintMetadata(CGF, E, Store, 1);
+ return Store;
+}
- if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
+static Value *EmitAtomicFetchWithHintBuiltin(CodeGenFunction &CGF,
+ const CallExpr *E,
+ AtomicRMWInst::BinOp Op) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ Expr::EvalResult Result;
+ if (!E->getArg(2)->EvaluateAsInt(Result, CGF.getContext()))
llvm_unreachable(
- "Expected integer hint argument to atomic store with hint.");
- unsigned HintArg = Result.Val.getInt().getExtValue();
+ "Expected integer policy argument to atomic fetch with hint.");
- // Attach the hint if valid
- if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::NONE) {
- LLVMContext &Ctx = CGM.getLLVMContext();
- MDNode *MemHint = MDNode::get(
- Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
- MDNode *HintNode = MDNode::get(
- CGM.getLLVMContext(),
- {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
+ AtomicOrdering Ordering;
+ unsigned OrderingArg = Result.Val.getInt().getExtValue();
+ assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
- Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
+ switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
+ case AtomicOrderingCABI::relaxed:
+ Ordering = AtomicOrdering::Monotonic;
+ break;
+ case AtomicOrderingCABI::consume:
+ case AtomicOrderingCABI::acquire:
+ Ordering = AtomicOrdering::Acquire;
+ break;
+ case AtomicOrderingCABI::release:
+ Ordering = AtomicOrdering::Release;
+ break;
+ case AtomicOrderingCABI::acq_rel:
+ Ordering = AtomicOrdering::AcquireRelease;
+ break;
+ case AtomicOrderingCABI::seq_cst:
+ Ordering = AtomicOrdering::SequentiallyConsistent;
+ break;
}
- return Store;
+ const Expr *Ptr = E->getArg(0);
+ Address Addr = CGF.EmitPointerWithAlignment(Ptr);
+ Addr = Addr.withElementType(
+ CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
+ const Expr *Data = E->getArg(1);
+ Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
+
+ AtomicRMWInst *RMW = Builder.CreateAtomicRMW(Op, Addr, DataVal, Ordering);
+ RMW->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
+ AttachAtomicHintMetadata(CGF, E, RMW, 0);
+ return CGF.EmitFromMemory(RMW, E->getType());
}
Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
@@ -4871,8 +4923,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
- if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ switch (BuiltinID) {
+ default:
+ break;
+ case AArch64::BI__builtin_arm_atomic_store_with_hint:
return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
+ case AArch64::BI__builtin_arm_atomic_fetch_add_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Add);
+ case AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Sub);
+ case AArch64::BI__builtin_arm_atomic_fetch_and_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::And);
+ case AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Xor);
+ case AArch64::BI__builtin_arm_atomic_fetch_or_with_hint:
+ return EmitAtomicFetchWithHintBuiltin(*this, E, AtomicRMWInst::Or);
+ }
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 395b760e7dbf09..cfbe490d995bfa 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -742,7 +742,7 @@ __arm_st64bv0(void *__addr, data512_t __value) {
}
#endif
-/* Atomic store with hints */
+/* Atomic intrinsics with hints */
#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
#define HINT_STSHH_KEEP 0
#define HINT_STSHH_STRM 1
@@ -751,6 +751,16 @@ __arm_st64bv0(void *__addr, data512_t __value) {
#define HINT_SHUH_PH 4
#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
__builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_add_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_sub_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_and_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_xor_with_hint(ptr, data, memory_order, hint)
+#define __arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_fetch_or_with_hint(ptr, data, memory_order, hint)
#endif
/* 11.1 Special register intrinsics */
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 2e8b90def62663..9938e187f7aad5 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -323,14 +323,16 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
return false;
}
-bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
- CallExpr *TheCall) {
+bool SemaARM::BuiltinARMAtomicHintCall(unsigned BuiltinID, CallExpr *TheCall) {
if (SemaRef.checkArgCount(TheCall, 4))
return true;
- // Arg 0 should be the pointer type. The pointee type must be a
- // scalar integral or floating-point type of 8, 16, 32 or 64 bits.
+ bool IsStore = BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint;
ASTContext &Context = getASTContext();
+
+ // Arg 0 should be the pointer type. The pointee type must be a
+ // scalar integral type of 8, 16, 32 or 64 bits. Stores also support
+ // floating-point types.
auto PtrArgRes =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
if (PtrArgRes.isInvalid())
@@ -340,7 +342,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_builtin_must_be_pointer)
- << PtrArg->getType() << 0 << PtrArg->getSourceRange();
+ << PtrArg->getType() << IsStore << PtrArg->getSourceRange();
TheCall->setArg(0, PtrArg);
QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType());
@@ -350,10 +352,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrQT << PtrArg->getSourceRange();
PtrQT = PtrQT.getUnqualifiedType();
- if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
- !PtrQT->isMFloat8Type())
+ if (!PtrQT->isIntegralType(Context) &&
+ !(IsStore && (PtrQT->isFloatingType() || PtrQT->isMFloat8Type())))
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_op_needs_atomic_int_or_fp)
+ IsStore ? diag::err_atomic_op_needs_atomic_int_or_fp
+ : diag::err_atomic_op_needs_atomic_int)
<< 0 << PtrQT << PtrArg->getSourceRange();
if (PtrQT->isBitIntType())
@@ -364,9 +367,12 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
- << PtrArg->getSourceRange();
+ << IsStore << PtrArg->getSourceRange();
+
+ if (!IsStore)
+ TheCall->setType(PtrQT);
- // Arg 1 is the data to be stored. The type must match the pointee
+ // Arg 1 is the data operand. The type must match the pointee
// type found above.
auto DataArgRes =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
@@ -382,7 +388,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_typecheck_call_
diff erent_arg_types)
<< PtrQT << DataQT;
- // Arg 2 is the memory order, which must be relaxed, release or seq_cst
+ // Arg 2 is the memory order. Stores only allow relaxed, release or seq_cst.
auto MemOrdArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2));
if (MemOrdArg.isInvalid())
@@ -407,7 +413,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< *MemOrdAP << MemOrd->getSourceRange();
auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
- if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ if (IsStore && AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
AtomicOrdering != llvm::AtomicOrderingCABI::release &&
AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
return Diag(TheCall->getBeginLoc(),
@@ -416,7 +422,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
}
// Arg 3 is the hint type. Only values represented by AArch64MemoryHint
- // are valid.
+ // are valid. Fetch operations only allow HINT_SHUH and HINT_SHUH_PH.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3));
if (HintArg.isInvalid())
@@ -433,8 +439,10 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_hint_has_invalid_hint_type)
<< Hint->getType() << Hint->getSourceRange();
- if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) ==
- llvm::AArch64MemoryHint::NONE) {
+ auto MemoryHint = llvm::toAArch64MemoryHint(HintAP->getZExtValue());
+ if (MemoryHint == llvm::AArch64MemoryHint::NONE ||
+ (!IsStore && MemoryHint != llvm::AArch64MemoryHint::SHUH &&
+ MemoryHint != llvm::AArch64MemoryHint::SHUH_PH)) {
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << Hint->getSourceRange();
return false;
@@ -1314,8 +1322,13 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
BuiltinID == AArch64::BI__builtin_arm_wsrp)
return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
- if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
- return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_add_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_sub_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_and_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_xor_with_hint ||
+ BuiltinID == AArch64::BI__builtin_arm_atomic_fetch_or_with_hint)
+ return BuiltinARMAtomicHintCall(BuiltinID, TheCall);
// Only check the valid encoding range. Any constant in this range would be
// converted to a register of the form S2_2_C3_C4_5. Let the hardware throw
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-hint-template.cpp
similarity index 66%
rename from clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
rename to clang/test/CodeGen/AArch64/atomic-hint-template.cpp
index ecf1e10a09b5da..0f64aacd6d9694 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-hint-template.cpp
@@ -55,9 +55,37 @@ void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) {
__builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{});
}
+template <unsigned Order>
+int test_atomic_fetch_hint_template_order(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr noundef i32 @_Z37test_atomic_fetch_hint_template_orderILj2EEiPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw add ptr [[PTR]], i32 [[VAL]] acquire, align 4, !mem.cache_hint [[META8:![0-9]+]]
+// CHECK-NEXT: ret i32 [[TMP0]]
+ return __arm_atomic_fetch_add_with_hint(ptr, val, Order, 3);
+}
+
+template int test_atomic_fetch_hint_template_order<__ATOMIC_ACQUIRE>(int *, int);
+
+template <unsigned Hint>
+int test_atomic_fetch_hint_template_hint(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr noundef i32 @_Z36test_atomic_fetch_hint_template_hintILj4EEiPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = atomicrmw xor ptr [[PTR]], i32 [[VAL]] seq_cst, align 4, !mem.cache_hint [[META10:![0-9]+]]
+// CHECK-NEXT: ret i32 [[TMP0]]
+ return __arm_atomic_fetch_xor_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
+}
+
+template int test_atomic_fetch_hint_template_hint<4>(int *, int);
+
//.
// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
// CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]}
// CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1}
+// CHECK: [[META8]] = !{i32 0, [[META9:![0-9]+]]}
+// CHECK: [[META9]] = !{!"aarch64.mem_hint", i32 3}
+// CHECK: [[META10]] = !{i32 0, [[META11:![0-9]+]]}
+// CHECK: [[META11]] = !{!"aarch64.mem_hint", i32 4}
//.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 60ec3e48e94980..2acb0373632c52 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1989,9 +1989,120 @@ void test_atomic_store_hint_array_arg() {
__builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM);
}
+// AArch64-LABEL: @test_atomic_fetch_hint_char(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i8 [[DATA:%.*]] monotonic, align 1, !mem.cache_hint [[FETCH0:![0-9]+]]
+// AArch64-NEXT: ret i8 [[OLD]]
+//
+char test_atomic_fetch_hint_char(char *ptr, char data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_uchar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i8 [[DATA:%.*]] release, align 1, !mem.cache_hint [[FETCH1:![0-9]+]]
+// AArch64-NEXT: ret i8 [[OLD]]
+//
+unsigned char test_atomic_fetch_hint_uchar(unsigned char *ptr, unsigned char data) {
+ return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELEASE, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_short(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw and ptr [[PTR:%.*]], i16 [[DATA:%.*]] acquire, align 2, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i16 [[OLD]]
+//
+short test_atomic_fetch_hint_short(short *ptr, short data) {
+ return __arm_atomic_fetch_and_with_hint(ptr, data, __ATOMIC_ACQUIRE, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_ushort(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i16 [[DATA:%.*]] acq_rel, align 2, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i16 [[OLD]]
+//
+unsigned short test_atomic_fetch_hint_ushort(unsigned short *ptr, unsigned short data) {
+ return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_ACQ_REL, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw or ptr [[PTR:%.*]], i32 [[DATA:%.*]] seq_cst, align 4, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_int(int *ptr, int data) {
+ return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_SEQ_CST, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_unsigned(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] acquire, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+unsigned test_atomic_fetch_hint_unsigned(unsigned *ptr, unsigned data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_CONSUME, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_long(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[PTR:%.*]], i64 [[DATA:%.*]] monotonic, align 8, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i64 [[OLD]]
+//
+long test_atomic_fetch_hint_long(long *ptr, long data) {
+ return __arm_atomic_fetch_sub_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_bool(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8
+// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0
+// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw xor ptr [[PTR:%.*]], i8 [[STOREDV1]] monotonic, align 1, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: [[LOADEDV1:%.*]] = icmp ne i8 [[OLD]], 0
+// AArch64-NEXT: ret i1 [[LOADEDV1]]
+//
+bool test_atomic_fetch_hint_bool(bool *ptr, bool data) {
+ return __arm_atomic_fetch_xor_with_hint(ptr, data, __ATOMIC_RELAXED, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_volatile(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw volatile or ptr [[PTR:%.*]], i32 [[DATA:%.*]] acq_rel, align 4, !mem.cache_hint [[FETCH0]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_volatile(volatile int *ptr, int data) {
+ return __arm_atomic_fetch_or_with_hint(ptr, data, __ATOMIC_ACQ_REL, 3);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_typedef(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[PTR:%.*]], i32 [[DATA:%.*]] release, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+aliased_int test_atomic_fetch_hint_typedef(aliased_int *ptr, const int data) {
+ return __arm_atomic_fetch_add_with_hint(ptr, data, __ATOMIC_RELEASE, 4);
+}
+
+// AArch64-LABEL: @test_atomic_fetch_hint_array_arg(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4
+// AArch64-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[STORAGE]], i8 0, i64 4, i1 false)
+// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0
+// AArch64-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[ARRAYDECAY]], i32 1 monotonic, align 4, !mem.cache_hint [[FETCH1]]
+// AArch64-NEXT: ret i32 [[OLD]]
+//
+int test_atomic_fetch_hint_array_arg(void) {
+ int storage[1] = {0};
+ return __arm_atomic_fetch_sub_with_hint(storage, 1, __ATOMIC_RELAXED, 4);
+}
+
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1}
+// AArch64: [[FETCH0]] = !{i32 0, [[FETCH_HINT0:![0-9]+]]}
+// AArch64-NEXT: [[FETCH_HINT0]] = !{!"aarch64.mem_hint", i32 3}
+// AArch64-NEXT: [[FETCH1]] = !{i32 0, [[FETCH_HINT1:![0-9]+]]}
+// AArch64-NEXT: [[FETCH_HINT1]] = !{!"aarch64.mem_hint", i32 4}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index b1002362585803..ed83126ce54203 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -242,6 +242,70 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 5); // Invalid Hint
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
+ // CHECK-NOT: !mem.cache_hint
+ // CHECK-NEXT: ret void
+}
+
+// CHECK-LABEL: @atomic_fetch_add_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F1:[0-9]+]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_add_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_sub_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw sub ptr {{.*}}, i64 {{.*}} release, align 8, !mem.cache_hint ![[F3:[0-9]+]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_sub_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELEASE, 4);
+}
+
+// CHECK-LABEL: @atomic_fetch_and_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw and ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_and_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_ACQUIRE, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_xor_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw xor ptr {{.*}}, i64 {{.*}} acq_rel, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_xor_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_xor_with_hint(a, b, __ATOMIC_ACQ_REL, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_or_with_hint(
+// CHECK: [[OLD:%.*]] = atomicrmw or ptr {{.*}}, i64 {{.*}} seq_cst, align 8, !mem.cache_hint ![[F3]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_or_with_hint(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_or_with_hint(a, b, __ATOMIC_SEQ_CST, 4);
+}
+
+// CHECK-LABEL: @atomic_fetch_with_hint_consume(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} acquire, align 8, !mem.cache_hint ![[F1]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_with_hint_consume(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_CONSUME, 3);
+}
+
+// CHECK-LABEL: @atomic_fetch_add_with_hint_ph(
+// CHECK: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8, !mem.cache_hint ![[F3]]
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_add_with_hint_ph(int64_t *a, int64_t b) {
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 4);
+}
+
+// CHECK-LABEL: @atomic_fetch_with_invalid_hint(
+// CHECK: atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: atomicrmw sub ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: atomicrmw and ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr {{.*}}, i64 {{.*}} monotonic, align 8{{$}}
+// CHECK-NEXT: ret i64 [[OLD]]
+int64_t atomic_fetch_with_invalid_hint(int64_t *a, int64_t b) {
+ __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+ __builtin_arm_atomic_fetch_sub_with_hint(a, b, __ATOMIC_RELAXED, HINT_STSHH_STRM);
+ __builtin_arm_atomic_fetch_and_with_hint(a, b, __ATOMIC_RELAXED, HINT_STCPH);
+ return __builtin_arm_atomic_fetch_add_with_hint(a, b, __ATOMIC_RELAXED, 5);
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
@@ -255,3 +319,5 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// CHECK: ![[M8]] = !{!"aarch64.mem_hint", i32 3}
// CHECK: ![[M9]] = !{i32 1, ![[M10:[0-9]+]]}
// CHECK: ![[M10]] = !{!"aarch64.mem_hint", i32 4}
+// CHECK: ![[F1]] = !{i32 0, ![[M8]]}
+// CHECK: ![[F3]] = !{i32 0, ![[M10]]}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index a79fb9177cec88..96774a02fca646 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -80,3 +80,33 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
// expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
}
+
+void test_atomic_fetch_hint(char *c_ptr, char c_data, const char *const_c_ptr,
+ float *f_ptr, float f_data, __int128 *wide_ptr,
+ __int128 wide_data, unsigned _BitInt(8) *bit_ptr,
+ unsigned _BitInt(8) bit_data, int variable,
+ int **ptr_ptr, _Atomic int *atomic_ptr) {
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 3, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+
+ __builtin_arm_atomic_fetch_and_with_hint(0, c_data, 0, 3); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral type of 8, 16, 32, or 64 bits ('int' invalid)}}
+ __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, f_data, 0, 3); // expected-error {{arguments are of
diff erent types ('char' vs 'float')}}
+ __builtin_arm_atomic_fetch_or_with_hint(const_c_ptr, c_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
+ __builtin_arm_atomic_fetch_add_with_hint(f_ptr, f_data, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('float' invalid)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(wide_ptr, wide_data, 0, 3); // expected-error {{address argument to atomic fetch with hint must be of size 8, 16, 32 or 64 bits}}
+ __builtin_arm_atomic_fetch_and_with_hint(bit_ptr, bit_data, 0, 3); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+ __builtin_arm_atomic_fetch_xor_with_hint(ptr_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('int *' invalid)}}
+ __builtin_arm_atomic_fetch_or_with_hint(atomic_ptr, 0, 0, 3); // expected-error {{address argument to atomic operation must be a pointer to integer ('_Atomic(int)' invalid)}}
+
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, variable, 3); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, -1, 3); // expected-error {{invalid memory order argument to atomic hint operation (-1 invalid)}}
+ __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 6, 3); // expected-error {{invalid memory order argument to atomic hint operation (6 invalid)}}
+
+ __builtin_arm_atomic_fetch_xor_with_hint(c_ptr, c_data, 0, variable); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_fetch_or_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
+ // expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 0); // expected-warning {{unrecognised hint type argument to atomic hint operation (0)}}
+ __builtin_arm_atomic_fetch_sub_with_hint(c_ptr, c_data, 0, 1); // expected-warning {{unrecognised hint type argument to atomic hint operation (1)}}
+ __builtin_arm_atomic_fetch_and_with_hint(c_ptr, c_data, 0, 2); // expected-warning {{unrecognised hint type argument to atomic hint operation (2)}}
+ __builtin_arm_atomic_fetch_add_with_hint(c_ptr, c_data, 0, 5); // expected-warning {{unrecognised hint type argument to atomic hint operation (5)}}
+}
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
index f47eb0c945d1b1..aefd1c10764623 100644
--- a/llvm/include/llvm/Support/AArch64MemoryHints.h
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -19,11 +19,6 @@ enum class AArch64MemoryHint {
SHUH_PH = 5,
};
-template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
- return (Int)AArch64MemoryHint::STSHH_KEEP <= I &&
- I <= (Int)AArch64MemoryHint::SHUH_PH;
-}
-
template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
switch (I) {
case 0:
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c3755d713f8aff..3320df7d951471 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -287,6 +287,9 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
+ // Emit expansion of atomic fetch with hint pseudo instructions
+ void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI);
+
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
EmitToStreamer(*OutStreamer, Inst);
@@ -3402,6 +3405,20 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
EmitToStreamer(*OutStreamer, Store);
}
+void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion(
+ const MachineInstr *MI) {
+ EmitToStreamer(
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
+
+ MCInst Fetch;
+ Fetch.setOpcode(MI->getOperand(3).getImm());
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+ Fetch.addOperand(MCOperand::createReg(MI->getOperand(2).getReg()));
+ Fetch.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Fetch);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -4125,6 +4142,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_X:
emitAtomicHintPseudoExpansion(MI);
return;
+ case AArch64::ATOMIC_FETCH_HINT_W:
+ case AArch64::ATOMIC_FETCH_HINT_X:
+ emitAtomicFetchHintPseudoExpansion(MI);
+ return;
case AArch64::ATOMIC_STORE_HINT_BroW:
case AArch64::ATOMIC_STORE_HINT_HroW:
case AArch64::ATOMIC_STORE_HINT_WroW:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 5befbf84d3bb8e..7157122b1d9358 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -34062,11 +34062,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
}
// The !mem.cache_hint metadata is not supported by GISel and will be dropped.
- // TODO: Remove this and handle atomic store hints in GISel once supported.
+ // TODO: Remove this and handle atomic hints in GISel once supported.
if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
return true;
}
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&Inst))
+ if (getMemCacheHintMetadata(*RMW))
+ return true;
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 1fa094edfbb82c..6564971876d9ec 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -283,7 +283,7 @@ def : Pat<(relaxed_store<atomic_store_64>
(STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
//===----------------------------------
-// Atomic store with hint pseudos
+// Atomic operations with hint pseudos
//===----------------------------------
let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
@@ -438,6 +438,66 @@ let AddedComplexity = 1, GISelShouldIgnore = 1 in {
defm : AtomicHintPatterns<0, 51, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>;
}
+// Keep the hint and the LSE instruction together until assembly emission.
+// The opcode operand lets all operations and orderings share these pseudos.
+let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayLoad = 1,
+ mayStore = 1 in {
+ class BaseFetchHintPseudo<RegisterClass dsttype, RegisterClass srctype>
+ : Pseudo<(outs dsttype:$result),
+ (ins srctype:$data, GPR64sp:$addr, i32imm:$opcode,
+ i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ def ATOMIC_FETCH_HINT_W : BaseFetchHintPseudo<GPR32common, GPR32>;
+ def ATOMIC_FETCH_HINT_X : BaseFetchHintPseudo<GPR64common, GPR64>;
+}
+
+class atomic_hint_fetch<PatFrag Base, code Pred>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (Base node:$ptr, node:$val), Pred>;
+
+// Pass the LSE opcode as an immediate so all operations and orderings can use
+// the same two fetch pseudos.
+class AtomicFetchOpcode<string Inst> : SDNodeXForm<imm,
+ "return CurDAG->getTargetConstant(AArch64::" # Inst # ", SDLoc(N), MVT::i32);">;
+
+multiclass AtomicFetchHintPatternsForOpcode<string Inst, string Op,
+ RegisterClass RC, Instruction Pseudo> {
+ defvar Base = !cast<PatFrag>(Op);
+ defvar Opcode = AtomicFetchOpcode<Inst>;
+ def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH); }]>
+ GPR64sp:$addr, RC:$data),
+ (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 50))>;
+ def : Pat<(atomic_hint_fetch<Base, [{ return isAtomicMemoryHint(N, AArch64MemoryHint::SHUH_PH); }]>
+ GPR64sp:$addr, RC:$data),
+ (Pseudo RC:$data, GPR64sp:$addr, (Opcode (i32 0)), (i32 51))>;
+}
+
+multiclass AtomicFetchHintPatternsOrd<string Inst, string Suffix, string Op,
+ RegisterClass RC, Instruction Pseudo> {
+ defm : AtomicFetchHintPatternsForOpcode<Inst # Suffix, Op # "_monotonic", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "A" # Suffix, Op # "_acquire", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "L" # Suffix, Op # "_release", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_acq_rel", RC, Pseudo>;
+ defm : AtomicFetchHintPatternsForOpcode<Inst # "AL" # Suffix, Op # "_seq_cst", RC, Pseudo>;
+}
+
+multiclass AtomicFetchHintPatterns<string Inst, string Op> {
+ defm : AtomicFetchHintPatternsOrd<Inst, "X", Op # "_i64", GPR64, ATOMIC_FETCH_HINT_X>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "W", Op # "_i32", GPR32, ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "H", Op # "_i16", GPR32, ATOMIC_FETCH_HINT_W>;
+ defm : AtomicFetchHintPatternsOrd<Inst, "B", Op # "_i8", GPR32, ATOMIC_FETCH_HINT_W>;
+}
+
+// SelectionDAG lowers SUB to ADD with a negated value and AND to CLR with a
+// complemented value. Hinted atomics use SelectionDAG, so separate SUB/AND
+// patterns are not needed here (unlike the GlobalISel patterns below).
+let Predicates = [HasLSE], AddedComplexity = 1 in {
+ defm : AtomicFetchHintPatterns<"LDADD", "atomic_load_add">;
+ defm : AtomicFetchHintPatterns<"LDSET", "atomic_load_or">;
+ defm : AtomicFetchHintPatterns<"LDEOR", "atomic_load_xor">;
+ defm : AtomicFetchHintPatterns<"LDCLR", "atomic_load_clr">;
+}
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
new file mode 100644
index 00000000000000..aae55ab0c25cbf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-fetch-hint.ll
@@ -0,0 +1,241 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
+
+;
+; SHUH, Relaxed
+;
+
+define i8 @test_atomic_fetch_add_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldaddb w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i8 %val monotonic, align 1, !mem.cache_hint !0
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_add_shuh_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldaddh w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i16 %val monotonic, align 2, !mem.cache_hint !0
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_add_shuh_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldadd w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw add ptr %ptr, i32 %val monotonic, align 4, !mem.cache_hint !0
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_add_shuh_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_add_shuh_relaxed_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldadd x1, x0, [x0]
+; CHECK-NEXT: ret
+; CHECK-
+ %old = atomicrmw add ptr %ptr, i64 %val monotonic, align 8, !mem.cache_hint !0
+ ret i64 %old
+}
+
+;
+; SHUH, AcquireRelease
+;
+
+define i8 @test_atomic_fetch_and_shuh_acq_rel_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclralb w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i8 %val acq_rel, align 1, !mem.cache_hint !0
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_and_shuh_acq_rel_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclralh w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i16 %val acq_rel, align 2, !mem.cache_hint !0
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_and_shuh_acq_rel_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn w8, w1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclral w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw and ptr %ptr, i32 %val acq_rel, align 4, !mem.cache_hint !0
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_and_shuh_acq_rel_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_and_shuh_acq_rel_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvn x8, x1
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldclral x8, x0, [x0]
+; CHECK-NEXT: ret
+
+ %old = atomicrmw and ptr %ptr, i64 %val acq_rel, align 8, !mem.cache_hint !0
+ ret i64 %old
+}
+
+;
+; SHUH, SequentiallyConsistent
+;
+
+; Sequentially consistent fetch-xor: retain acquire semantics when the
+; result is unused.
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoralb w1, w8, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i8 %val seq_cst, align 1, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoralh w1, w8, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i16 %val seq_cst, align 2, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoral w1, w8, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i32 %val seq_cst, align 4, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_fetch_xor_shuh_seqcst_dead_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_xor_shuh_seqcst_dead_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh
+; CHECK-NEXT: ldeoral x1, x8, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw xor ptr %ptr, i64 %val seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; SHUH PH, Acquire
+;
+
+define i8 @test_atomic_fetch_sub_shuh_ph_acquire_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldaddab w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i8 %val acquire, align 1, !mem.cache_hint !2
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_sub_shuh_ph_acquire_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldaddah w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i16 %val acquire, align 2, !mem.cache_hint !2
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_sub_shuh_ph_acquire_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg w8, w1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldadda w8, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw sub ptr %ptr, i32 %val acquire, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_sub_shuh_ph_acquire_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_sub_shuh_ph_acquire_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: neg x8, x1
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldadda x8, x0, [x0]
+; CHECK-NEXT: ret
+
+ %old = atomicrmw sub ptr %ptr, i64 %val acquire, align 8, !mem.cache_hint !2
+ ret i64 %old
+}
+
+;
+; SHUH PH, Release
+;
+
+define i8 @test_atomic_fetch_or_shuh_ph_release_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetlb w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i8 %val release, align 1, !mem.cache_hint !2
+ ret i8 %old
+}
+
+define i16 @test_atomic_fetch_or_shuh_ph_release_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetlh w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i16 %val release, align 2, !mem.cache_hint !2
+ ret i16 %old
+}
+
+define i32 @test_atomic_fetch_or_shuh_ph_release_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetl w1, w0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i32 %val release, align 4, !mem.cache_hint !2
+ ret i32 %old
+}
+
+define i64 @test_atomic_fetch_or_shuh_ph_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_fetch_or_shuh_ph_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shuh ph
+; CHECK-NEXT: ldsetl x1, x0, [x0]
+; CHECK-NEXT: ret
+ %old = atomicrmw or ptr %ptr, i64 %val release, align 8, !mem.cache_hint !2
+ ret i64 %old
+}
+
+!0 = !{i32 0, !1}
+!1 = !{!"aarch64.mem_hint", i32 3}
+!2 = !{i32 0, !3}
+!3 = !{!"aarch64.mem_hint", i32 4}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index 2528a4079b0320..06dc124c79088a 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -457,7 +457,7 @@ define void @test_atomic_store_stcph_seqcst_i64(ptr %ptr, i64 %val) nounwind {
define void @test_atomic_store_shuh_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_relaxed_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: strb w1, [x0]
; CHECK-NEXT: ret
store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !6
@@ -531,7 +531,7 @@ define void @test_atomic_store_shuh_release_i32(ptr %ptr, i32 %val) nounwind {
define void @test_atomic_store_shuh_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_shuh_release_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: shuh{{$}}
+; CHECK-NEXT: shuh
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !6
More information about the cfe-commits
mailing list