[llvm] 34f5f2c - [AArch64] Implement the atomic store with hint intrinsic (#198316)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 06:26:46 PDT 2026
Author: Kerry McLaughlin
Date: 2026-09-24T14:26:38+01:00
New Revision: 34f5f2c594111e6a3edf65673c3ea2849e96e2cb
URL: https://github.com/llvm/llvm-project/commit/34f5f2c594111e6a3edf65673c3ea2849e96e2cb
DIFF: https://github.com/llvm/llvm-project/commit/34f5f2c594111e6a3edf65673c3ea2849e96e2cb.diff
LOG: [AArch64] Implement the atomic store with hint intrinsic (#198316)
Adds the following ACLE intrinsic as described in [1]:
void __arm_atomic_store_with_hint(type *ptr, type data,
int memory_order, int hint);
An atomic store instruction is emitted in Clang for this builtin with additional
metadata attached with !mem.cache_hint, which ensures the instruction is
recognised as atomic by later passes in LLVM. When an atomic store is found
with this metadata, it is lowered to the ATOMIC_STORE_HINT pseudo which is
later expanded into an STSHH instruction plus an atomic store.
The hint is a suggestion to the compiler and maps directly to a specific hint
instruction variant in the ISA. The compiler may use this hint when selecting
code sequences, but it is not required to emit a specific hint instruction or a
specific instruction sequence.
Supported atomic store hints are currently:
HINT_STSHH_KEEP: Requests retention of the updated location in the
local cache of the updating PE.
HINT_STSHH_STRM: Requests that the updated location not be retained
in the local cache of the updating PE.
[1] https://github.com/ARM-software/acle/pull/432
Added:
clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
llvm/include/llvm/Support/AArch64MemoryHints.h
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
Modified:
clang/include/clang/Basic/BuiltinsAArch64.td
clang/include/clang/Basic/DiagnosticSemaKinds.td
clang/include/clang/Sema/Sema.h
clang/include/clang/Sema/SemaARM.h
clang/lib/CodeGen/TargetBuiltins/ARM.cpp
clang/lib/Headers/arm_acle.h
clang/lib/Sema/SemaARM.cpp
clang/lib/Sema/SemaChecking.cpp
clang/test/CodeGen/arm_acle.c
clang/test/CodeGen/builtins-arm64.c
clang/test/Sema/builtins-arm64.c
llvm/lib/CodeGen/AtomicExpandPass.cpp
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64InstrAtomics.td
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
llvm/lib/Target/AArch64/AArch64InstrInfo.h
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 4c49c874133fb6..30aa3d526cbbb7 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -171,6 +171,10 @@ let Attributes = [NoThrow], Features = "ls64" in {
def st64bv0 : AArch64TargetBuiltin<"uint64_t (void *, uint64_t const *)">;
}
+let Attributes = [NoThrow, CustomTypeChecking] in {
+ def atomic_store_with_hint : AArch64Builtin<"void(...)">;
+}
+
// Armv9.3-A Guarded Control Stack
let Attributes = [NoThrow], Features = "gcs" in {
def gcspopm : AArch64TargetBuiltin<"uint64_t (uint64_t)">;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 36a18473f4d4cc..e854652a9efb85 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9615,6 +9615,13 @@ def err_atomic_builtin_must_be_pointer : Error<
def err_atomic_builtin_must_be_pointer_intptr : Error<
"address argument to atomic builtin must be a pointer to integer or pointer"
" (%0 invalid)">;
+
+def err_atomic_hint_builtin_must_be_pointer : Error<
+ "address argument to atomic hint builtin must be a pointer to a scalar "
+ "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+def err_atomic_hint_has_invalid_hint_type : Error<
+ "invalid hint type argument to atomic hint operation (%0)">;
+
def err_atomic_builtin_cannot_be_const : Error<
"address argument to atomic builtin cannot be const-qualified (%0 invalid)">;
def err_atomic_builtin_must_be_pointer_intfltptr : Error<
@@ -9683,6 +9690,13 @@ def err_atomic_op_needs_atomic_int : Error<
def err_atomic_op_needs_atomic_fp
: Error<"address argument to atomic operation must be a pointer to "
"%select{|atomic }0floating point type (%1 invalid)">;
+def err_atomic_op_hint_data_size : Error<
+ "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
+def err_atomic_hint_has_invalid_memory_order : Error<
+ "invalid memory order argument to atomic hint operation (%0 invalid)">;
+def warn_atomic_hint_has_invalid_hint_type : Warning<
+ "unrecognised hint type argument to atomic hint operation (%0)">,
+ InGroup<DiagGroup<"atomic-hint-type">>;
def warn_atomic_op_has_invalid_memory_order : Warning<
"%select{|success |failure }0memory order argument to atomic operation is invalid">,
InGroup<DiagGroup<"atomic-memory-ordering">>;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5f24adbf9e9cf3..eb3b908351448d 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2915,6 +2915,8 @@ class Sema final : public SemaBase {
/// This is useful when doing custom type-checking. Returns true on error.
bool checkArgCount(CallExpr *Call, unsigned DesiredArgCount);
+ bool convertArgumentToType(Expr *&Value, QualType Ty);
+
/// Returns true if the argument consists of one contiguous run of 1s with any
/// number of 0s on either side. The 1s are allowed to wrap from LSB to MSB,
/// so 0x000FFF0, 0x0000FFFF, 0xFF0000FF, 0x0 are all runs. 0x0F0F0000 is not,
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index af8e0e9047171d..b0a01c40ffecef 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -70,6 +70,7 @@ class SemaARM : public SemaBase {
bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum,
unsigned ExpectedFieldNum, bool AllowName);
bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall);
bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 621e5d18ebfeeb..52a7564789fb77 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -20,6 +20,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include <numeric>
@@ -2065,6 +2066,67 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
Ops);
}
+static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
+ unsigned BuiltinID,
+ const CallExpr *E) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ CodeGen::CodeGenModule &CGM = CGF.CGM;
+ Expr::EvalResult Result;
+ if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer policy argument to atomic store with hint.");
+
+ const Expr *Ptr = E->getArg(0);
+ Address Addr = CGF.EmitPointerWithAlignment(Ptr);
+ Addr = Addr.withElementType(
+ CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
+
+ const Expr *Data = E->getArg(1);
+ Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
+
+ StoreInst *Store = Builder.CreateStore(DataVal, Addr);
+ Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
+
+ AtomicOrdering Ordering;
+ unsigned OrderingArg = Result.Val.getInt().getExtValue();
+ assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
+
+ switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
+ default:
+ llvm_unreachable("Unsupported atomic ordering found.");
+ case AtomicOrderingCABI::relaxed:
+ Ordering = AtomicOrdering::Monotonic;
+ break;
+ case AtomicOrderingCABI::release:
+ Ordering = AtomicOrdering::Release;
+ break;
+ case AtomicOrderingCABI::seq_cst:
+ Ordering = AtomicOrdering::SequentiallyConsistent;
+ break;
+ }
+ Store->setAtomic(Ordering);
+
+ if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer hint argument to atomic store with hint.");
+ unsigned HintArg = Result.Val.getInt().getExtValue();
+
+ // Attach the hint if valid
+ if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::HINT_NONE) {
+ LLVMContext &Ctx = CGM.getLLVMContext();
+ MDNode *MemHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
+ MDNode *HintNode = MDNode::get(
+ CGM.getLLVMContext(),
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
+
+ Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
+ }
+
+ return Store;
+}
+
/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
/// argument that specifies the vector type. The additional argument is meant
/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
@@ -4840,6 +4902,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
+
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 73b1e841b6719d..715fe851b0c322 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -742,6 +742,14 @@ __arm_st64bv0(void *__addr, data512_t __value) {
}
#endif
+/* Atomic store with hints */
+#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
+#define HINT_STSHH_KEEP 0
+#define HINT_STSHH_STRM 1
+#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
+#endif
+
/* 11.1 Special register intrinsics */
#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f38a35ec08fb0a..2bf6901bcc07f2 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -17,6 +17,7 @@
#include "clang/Sema/Initialization.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
+#include "llvm/Support/AArch64MemoryHints.h"
namespace clang {
@@ -322,6 +323,126 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
return false;
}
+bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
+ CallExpr *TheCall) {
+ if (SemaRef.checkArgCount(TheCall, 4))
+ return true;
+
+ // Arg 0 should be the pointer type. The pointee type must be a
+ // scalar integral or floating-point type of 8, 16, 32 or 64 bits.
+ ASTContext &Context = getASTContext();
+ auto PtrArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
+ if (PtrArgRes.isInvalid())
+ return true;
+ auto *PtrArg = PtrArgRes.get();
+ auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
+ if (!PtrTy)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_builtin_must_be_pointer)
+ << PtrArg->getType() << 0 << PtrArg->getSourceRange();
+ TheCall->setArg(0, PtrArg);
+
+ QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType());
+ if (PtrQT.isConstQualified())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_non_const_pointer)
+ << PtrQT << PtrArg->getSourceRange();
+
+ PtrQT = PtrQT.getUnqualifiedType();
+ if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
+ !PtrQT->isMFloat8Type())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_atomic_int_or_fp)
+ << 0 << PtrQT << PtrArg->getSourceRange();
+
+ if (PtrQT->isBitIntType())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_builtin_bit_int_prohibit)
+ << PtrQT << PtrArg->getSourceRange();
+
+ unsigned TySize = Context.getTypeSize(PtrQT);
+ if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
+ return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
+ << PtrArg->getSourceRange();
+
+ // Arg 1 is the data to be stored. The type must match the pointee
+ // type found above.
+ auto DataArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
+ if (DataArgRes.isInvalid())
+ return true;
+ auto *DataArg = DataArgRes.get();
+ QualType DataQT =
+ Context.getCanonicalType(DataArg->getType()).getUnqualifiedType();
+ TheCall->setArg(1, DataArg);
+
+ if (PtrQT != DataQT)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_typecheck_call_
diff erent_arg_types)
+ << PtrQT << DataQT;
+
+ // Arg 2 is the memory order, which must be relaxed, release or seq_cst
+ auto MemOrdArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2));
+ if (MemOrdArg.isInvalid())
+ return true;
+ auto *MemOrd = MemOrdArg.get();
+ if (SemaRef.convertArgumentToType(MemOrd, Context.IntTy))
+ return true;
+ TheCall->setArg(2, MemOrd);
+
+ if (!MemOrd->isValueDependent()) {
+ std::optional<llvm::APSInt> MemOrdAP =
+ MemOrd->getIntegerConstantExpr(Context);
+ if (!MemOrdAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << MemOrd->getType() << MemOrd->getSourceRange();
+
+ unsigned Ordering = MemOrdAP->getZExtValue();
+ if (!llvm::isValidAtomicOrderingCABI(Ordering))
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrd->getSourceRange();
+
+ auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
+ if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::release &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrd->getSourceRange();
+ }
+
+ // Arg 3 is the hint type. Only values represented by AArch64MemoryHint
+ // are valid.
+ auto HintArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3));
+ if (HintArg.isInvalid())
+ return true;
+ auto Hint = HintArg.get();
+ if (SemaRef.convertArgumentToType(Hint, Context.IntTy))
+ return true;
+ TheCall->setArg(3, Hint);
+
+ if (!Hint->isValueDependent()) {
+ std::optional<llvm::APSInt> HintAP = Hint->getIntegerConstantExpr(Context);
+ if (!HintAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << Hint->getType() << Hint->getSourceRange();
+
+ if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) ==
+ llvm::AArch64MemoryHint::HINT_NONE) {
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << *HintAP << Hint->getSourceRange();
+ return false;
+ }
+ }
+ return false;
+}
+
/// getNeonEltType - Return the QualType corresponding to the elements of
/// the vector type specified by the NeonTypeFlags. This is used to check
/// the pointer arguments for Neon load/store intrinsics.
@@ -1170,6 +1291,9 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
BuiltinID == AArch64::BI__builtin_arm_wsrp)
return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall);
+
// Only check the valid encoding range. Any constant in this range would be
// converted to a register of the form S2_2_C3_C4_5. Let the hardware throw
// an exception for incorrect registers. This matches MSVC behavior.
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e4cb8ba0885d09..687af75c1496be 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -201,14 +201,14 @@ static bool checkBuiltinVerboseTrap(CallExpr *Call, Sema &S) {
return !HasError;
}
-static bool convertArgumentToType(Sema &S, Expr *&Value, QualType Ty) {
+bool Sema::convertArgumentToType(Expr *&Value, QualType Ty) {
if (Value->isTypeDependent())
return false;
InitializedEntity Entity =
- InitializedEntity::InitializeParameter(S.Context, Ty, false);
+ InitializedEntity::InitializeParameter(Context, Ty, false);
ExprResult Result =
- S.PerformCopyInitialization(Entity, SourceLocation(), Value);
+ PerformCopyInitialization(Entity, SourceLocation(), Value);
if (Result.isInvalid())
return true;
Value = Result.get();
@@ -1702,7 +1702,7 @@ static bool checkPointerAuthEnabled(Sema &S, Expr *E) {
static bool checkPointerAuthKey(Sema &S, Expr *&Arg) {
// Convert it to type 'int'.
- if (convertArgumentToType(S, Arg, S.Context.IntTy))
+ if (S.convertArgumentToType(Arg, S.Context.IntTy))
return true;
// Value-dependent expressions are okay; wait for template instantiation.
@@ -1838,7 +1838,7 @@ static bool checkPointerAuthValue(Sema &S, Expr *&Arg, PointerAuthOpKind OpKind,
// Convert to that type. This should just be an lvalue-to-rvalue
// conversion.
- if (convertArgumentToType(S, Arg, ExpectedTy))
+ if (S.convertArgumentToType(Arg, ExpectedTy))
return true;
if (!RequireConstant) {
@@ -6710,7 +6710,7 @@ bool Sema::BuiltinPrefetch(CallExpr *TheCall) {
// Argument 0 is checked for us and the remaining arguments must be
// constant integers.
for (unsigned i = 1; i != NumArgs; ++i) {
- if (convertArgumentToType(*this, TheCall->getArgs()[i], Context.IntTy))
+ if (convertArgumentToType(TheCall->getArgs()[i], Context.IntTy))
return true;
if (BuiltinConstantArgRange(TheCall, i, 0, i == 1 ? 1 : 3))
return true;
@@ -6825,7 +6825,7 @@ bool Sema::BuiltinAssumeAligned(CallExpr *TheCall) {
if (NumArgs > 2) {
Expr *ThirdArg = TheCall->getArg(2);
- if (convertArgumentToType(*this, ThirdArg, Context.getSizeType()))
+ if (convertArgumentToType(ThirdArg, Context.getSizeType()))
return true;
TheCall->setArg(2, ThirdArg);
}
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
new file mode 100644
index 00000000000000..ecf1e10a09b5da
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -0,0 +1,63 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 6
+// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-elf -target-feature +v9.4a -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+
+#include <arm_acle.h>
+
+// REQUIRES: aarch64-registered-target
+
+template <unsigned Order>
+void test_atomic_store_hint_template_order(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr void @_Z37test_atomic_store_hint_template_orderILj3EEvPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] release, align 4, !mem.cache_hint [[META4:![0-9]+]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, val, Order, 0);
+}
+
+template void test_atomic_store_hint_template_order<__ATOMIC_RELEASE>(int *, int);
+
+template <unsigned Hint>
+void test_atomic_store_hint_template_hint(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr void @_Z36test_atomic_store_hint_template_hintILj1EEvPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] seq_cst, align 4, !mem.cache_hint [[META6:![0-9]+]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
+}
+
+template void test_atomic_store_hint_template_hint<HINT_STSHH_STRM>(int *, int);
+
+struct MemVal {
+ constexpr operator int() const { return __ATOMIC_RELAXED; }
+};
+
+void test_atomic_store_hint_array_consexpr_order(int *ptr, int value) {
+// CHECK-LABEL: define dso_local void @_Z43test_atomic_store_hint_array_consexpr_orderPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META4]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, value, MemVal{}, 0);
+}
+
+struct HintVal {
+ constexpr operator int() const { return 1; }
+};
+
+void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) {
+// CHECK-LABEL: define dso_local void @_Z42test_atomic_store_hint_array_consexpr_hintPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META6]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{});
+}
+
+//.
+// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
+// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
+// CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]}
+// CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1}
+//.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index e1bbc3d745bcd0..60ec3e48e94980 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -8,6 +8,7 @@
#include <arm_acle.h>
+#include <stdbool.h>
// REQUIRES: arm-registered-target,aarch64-registered-target
@@ -1761,3 +1762,236 @@ int test_rndrrs(uint64_t *__addr) {
return __rndrrs(__addr);
}
#endif
+
+#if defined(__ARM_64BIT_STATE)
+
+// AArch64-LABEL: @test_atomic_store_hint_char(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_char(char *ptr, char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_uchar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_schar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_bool(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8
+// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0
+// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8
+// AArch64-NEXT: store atomic i8 [[STOREDV1]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_bool(bool *ptr, bool data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_mfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic <1 x i8> [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_mfloat(__mfp8 *ptr, __mfp8 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_bfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_half(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_STRM);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_short(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_short(short *ptr, short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_ushort(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_int(int *ptr, int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_unsigned(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_u32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_s32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_float(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_float(float *ptr, float data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_s64(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_long(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long(long *ptr, long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_ulong(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_double(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_double(double *ptr, double data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM);
+}
+
+typedef int aliased_int;
+
+// AArch64-LABEL: @test_atomic_store_hint_typedef_ptr(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] monotonic, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_typedef_ptr(aliased_int *ptr, int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_typedef_val(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] release, align 4, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_typedef_val(int *ptr, aliased_int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELEASE, HINT_STSHH_STRM);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_volatile(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic volatile i32 [[VALUE:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_volatile(volatile int *ptr, int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_array_arg(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4
+// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0
+// AArch64-NEXT: store atomic i32 0, ptr [[ARRAYDECAY]] monotonic, align 4, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_array_arg() {
+ int storage[1];
+ __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM);
+}
+
+// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
+// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
+
+// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
+// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1}
+#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 1800ff77a57a52..fb297cd3eec2c7 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -2,6 +2,7 @@
// RUN: %clang_cc1 -triple aarch64-windows -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-WIN
// RUN: %clang_cc1 -triple arm64_32-apple-ios13 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
#include <stdint.h>
+#include <arm_acle.h>
void f0(void *a, void *b) {
__clear_cache(a,b);
@@ -216,4 +217,26 @@ void trap() {
__builtin_arm_trap(42);
}
+void atomic_store_with_hint(int64_t *a, int64_t b) {
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M1:[0-9]]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M1]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
+
+ // Invalid hint should be dropped
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
+}
+
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
+// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
+// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0}
+// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
+// CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 41cffd7ebb1a0d..9a6e0715c4c788 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -1,6 +1,8 @@
// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify %s
// RUN: %clang_cc1 -triple arm64-apple-ios -DTEST1 -fsyntax-only -verify %s
+#include <arm_acle.h>
+
#ifdef TEST1
void __clear_cache(void *start, void *end);
#endif
@@ -51,3 +53,29 @@ void test_trap(short s, unsigned short us) {
__builtin_arm_trap(s); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
__builtin_arm_trap(us); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
}
+
+void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
+ char c_data, __int128 inv_data, float f_data,
+ int inv_int, const char *const_c_ptr,
+ unsigned _BitInt(7) *bit_ptr, unsigned _BitInt(7) *bit_data) {
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+
+ __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral or floating-point type of 8, 16, 32, or 64 bits ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of
diff erent types ('char' vs 'float')}}
+ __builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
+
+ __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
+
+ __builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, HINT_STSHH_STRM); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_INVALID); // expected-error {{use of undeclared identifier 'HINT_STSHH_INVALID'}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
+ // expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
+}
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
new file mode 100644
index 00000000000000..44ac64e481e80b
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -0,0 +1,35 @@
+//===-- AArch64MemoryHints.h - AArch64 Memory Hint Attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+#define LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+
+namespace llvm {
+enum class AArch64MemoryHint {
+ HINT_NONE = 0,
+ HINT_STSHH_KEEP = 1,
+ HINT_STSHH_STRM = 2,
+};
+
+template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
+ return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I &&
+ I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM;
+}
+
+template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
+ switch (I) {
+ case 0:
+ return AArch64MemoryHint::HINT_STSHH_KEEP;
+ case 1:
+ return AArch64MemoryHint::HINT_STSHH_STRM;
+ default:
+ return AArch64MemoryHint::HINT_NONE;
+ }
+}
+} // namespace llvm
+#endif // LLVM_SUPPORT_AARCH64MEMORYHINTS_H
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 50e09e40f6c757..b3d186fdf5f011 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -246,6 +246,7 @@ static void copyMetadataForAtomic(Instruction &Dest,
case LLVMContext::MD_tbaa:
case LLVMContext::MD_tbaa_struct:
case LLVMContext::MD_alias_scope:
+ case LLVMContext::MD_mem_cache_hint:
case LLVMContext::MD_noalias:
case LLVMContext::MD_noalias_addrspace:
case LLVMContext::MD_access_group:
@@ -759,6 +760,7 @@ StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
Value *Addr = SI->getPointerOperand();
StoreInst *NewSI = Builder.CreateStore(NewVal, Addr, SI->getProperties());
+ copyMetadataForAtomic(*NewSI, *SI);
LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
SI->eraseFromParent();
return NewSI;
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index bc8c5510ea5154..5a502f785e0f40 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -290,6 +290,11 @@ class AArch64AsmPrinter : public AsmPrinter {
// Emit expansion of Compare-and-branch pseudo instructions
void emitCBPseudoExpansion(const MachineInstr *MI);
+ // Emit expansion of atomic store with hint pseudo instructions
+ void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
+
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
EmitToStreamer(*OutStreamer, Inst);
@@ -3249,6 +3254,161 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, Inst);
}
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
+
+ unsigned StOpc;
+ unsigned Relaxed = MI->getOperand(2).getImm();
+ assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_B:
+ StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_H:
+ StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_W:
+ StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_X:
+ StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint size.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(3).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+ Store.setFlags(MI->getFlags());
+ if (Relaxed)
+ Store.addOperand(MCOperand::createImm(0));
+ EmitToStreamer(*OutStreamer, Store);
+}
+
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ assert(MI->getOperand(5).getImm() == 1 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ StOpc = AArch64::STRBBroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ StOpc = AArch64::STRHHroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_WroW:
+ StOpc = AArch64::STRWroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_XroW:
+ StOpc = AArch64::STRXroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ StOpc = AArch64::STRSroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ StOpc = AArch64::STRDroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ StOpc = AArch64::STRBBroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ StOpc = AArch64::STRHHroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_WroX:
+ StOpc = AArch64::STRWroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_XroX:
+ StOpc = AArch64::STRXroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ StOpc = AArch64::STRSroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ StOpc = AArch64::STRDroX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(6).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm
+ Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
+ Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed
+ Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ assert(MI->getOperand(3).getImm() == 1 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ StOpc = AArch64::STRBBui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ StOpc = AArch64::STRHHui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ StOpc = AArch64::STRWui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ StOpc = AArch64::STRXui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ StOpc = AArch64::STRSui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ StOpc = AArch64::STRDui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ StOpc = AArch64::STURBBi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ StOpc = AArch64::STURHHi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ StOpc = AArch64::STURWi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ StOpc = AArch64::STURXi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ StOpc = AArch64::STURSi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ StOpc = AArch64::STURDi;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
+ Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -3966,6 +4126,40 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::CBXPrr:
emitCBPseudoExpansion(MI);
return;
+ case AArch64::ATOMIC_STORE_HINT_B:
+ case AArch64::ATOMIC_STORE_HINT_H:
+ case AArch64::ATOMIC_STORE_HINT_W:
+ case AArch64::ATOMIC_STORE_HINT_X:
+ emitAtomicHintPseudoExpansion(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ case AArch64::ATOMIC_STORE_HINT_WroW:
+ case AArch64::ATOMIC_STORE_HINT_XroW:
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ case AArch64::ATOMIC_STORE_HINT_WroX:
+ case AArch64::ATOMIC_STORE_HINT_XroX:
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ emitAtomicHintPseudoExpansionRO(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ emitAtomicHintPseudoExpansionImm(MI);
+ return;
}
if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index feb8f465837940..ee87070c127272 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -23,6 +23,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
@@ -516,6 +517,10 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
+ AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
+ bool isAtomicSTSHH_KEEP(SDNode *N) const;
+ bool isAtomicSTSHH_STRM(SDNode *N) const;
+
bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
bool Negate);
bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
@@ -4614,6 +4619,34 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
+AArch64MemoryHint
+AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
+ int MemoryHint = -1;
+ const MDNode *MemCacheHint = MMO->getMemCacheHint();
+ if (!MemCacheHint)
+ return AArch64MemoryHint::HINT_NONE;
+
+ for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
+ if (MemCacheHint->getOperand(I).equalsStr("aarch64.mem_hint")) {
+ const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
+ MemoryHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
+ ->getZExtValue();
+ }
+ }
+
+ return toAArch64MemoryHint(MemoryHint);
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_KEEP;
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_STRM;
+}
+
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
SDValue &Shift, bool Negate) {
if (!isa<ConstantSDNode>(N))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d90030ee5519e7..d5135b47f4ef19 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -41,6 +41,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
@@ -34068,6 +34069,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
return true;
}
}
+
+ // The !mem.cache_hint metadata is not supported by GISel and will be dropped.
+ // TODO: Remove this and handle atomic store hints in GISel once supported.
+ if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
+ if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
+ return true;
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2187f21abb70fd..7e0e75c7a3c6b8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -282,6 +282,156 @@ def : Pat<(relaxed_store<atomic_store_64>
(am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
(STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
+//===----------------------------------
+// Atomic store with hint pseudos
+//===----------------------------------
+
+let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
+ class BaseStoreHintPseudo<RegisterClass regtype>
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$addr,
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR32:$Rm, WExt:$extend,
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR64:$Rm, XExt:$extend,
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, UImmOp:$offset,
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoImm<RegisterClass regtype>
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, simm9:$offset,
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+}
+
+multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> {
+ def NAME : BaseStoreHintPseudo<regtype>;
+ def NAME # roW : BaseStoreHintPseudoROW<regtype, WExt>;
+ def NAME # roX : BaseStoreHintPseudoROX<regtype, XExt>;
+ def NAME # ui : BaseStoreHintPseudoUImm<regtype, UImmOp>;
+ def NAME # i : BaseStoreHintPseudoImm<regtype>;
+}
+
+defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>;
+defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
+defm ATOMIC_STORE_HINT_W : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_X : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<FPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<FPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+
+class atomic_hint_store<PatFrag Base, int Relaxed, code Pred>
+ : PatFrag<(ops node:$val, node:$ptr),
+ (Base node:$val, node:$ptr), Pred> {
+ let IsAtomic = 1;
+ let IsAtomicOrderingReleaseOrStronger = !eq(Relaxed, 0);
+
+ // TODO: Change once mem.cache_hint supported in GISel
+ let GISelPredicateCode = [{ return false; }];
+}
+
+multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
+ GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
+ GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_WroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_XroW GPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW FPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW FPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
+ GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
+ GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_WroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_XroX GPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX FPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX FPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
+ GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
+ (ATOMIC_STORE_HINT_Bui GPR32:$data, GPR64sp:$addr, uimm12s1:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
+ GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
+ (ATOMIC_STORE_HINT_Hui GPR32:$data, GPR64sp:$addr, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Wui GPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Xui GPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui FPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui FPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
+ GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Bi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
+ GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Hi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Wi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Xi GPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si FPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di FPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+}
+
+multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_B GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_H GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_W GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_X GPR64:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+}
+
+let AddedComplexity = 15 in {
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
+
+ defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+}
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index b8e74f8ad496d9..6962d66dedaf01 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3151,6 +3151,18 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
case AArch64::STZ2Gi:
case AArch64::STZGi:
case AArch64::TAGPstack:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
return 2;
case AArch64::LD1B_D_IMM:
case AArch64::LD1B_H_IMM:
@@ -4761,6 +4773,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STRXui:
case AArch64::STRDui:
case AArch64::PRFMui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
Scale = Width = TypeSize::getFixed(8);
MinOffset = 0;
MaxOffset = 4095;
@@ -4770,6 +4784,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSWui:
case AArch64::STRWui:
case AArch64::STRSui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
Scale = Width = TypeSize::getFixed(4);
MinOffset = 0;
MaxOffset = 4095;
@@ -4780,6 +4796,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSHXui:
case AArch64::STRHui:
case AArch64::STRHHui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
Scale = Width = TypeSize::getFixed(2);
MinOffset = 0;
MaxOffset = 4095;
@@ -4790,6 +4807,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSBXui:
case AArch64::STRBui:
case AArch64::STRBBui:
+ case AArch64::ATOMIC_STORE_HINT_Bui:
Scale = Width = TypeSize::getFixed(1);
MinOffset = 0;
MaxOffset = 4095;
@@ -4868,6 +4886,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURDi:
case AArch64::STLURXi:
case AArch64::PRFUMi:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_Di:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(8);
MinOffset = -256;
@@ -4881,6 +4901,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURWi:
case AArch64::STURSi:
case AArch64::STLURWi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(4);
MinOffset = -256;
@@ -4896,6 +4918,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURHi:
case AArch64::STURHHi:
case AArch64::STLURHi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(2);
MinOffset = -256;
@@ -4911,6 +4934,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURBi:
case AArch64::STURBBi:
case AArch64::STLURBi:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
Scale = Width = TypeSize::getFixed(1);
MinOffset = -256;
MaxOffset = 255;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 99e0142824d438..6a8346395eea1a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -16,6 +16,7 @@
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/TypeSize.h"
#include <optional>
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
new file mode 100644
index 00000000000000..bfa9fc49cf7522
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -0,0 +1,339 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
+
+;
+; STSHH: Keep, Relaxed
+;
+
+define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, Release
+;
+
+define void @test_atomic_store_keep_release_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, SequentiallyConsistent
+;
+
+define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; STSHH: Stream, Relaxed
+;
+
+define void @test_atomic_store_strm_relaxed_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str s0, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str d0, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+;
+; STSHH: Stream, Release
+;
+
+define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !2
+ ret void
+}
+
+;
+; STSHH: Stream, SequentiallyConsistent
+;
+
+define void @test_atomic_store_stream_seqcst_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+;
+; Invalid Hint
+;
+
+define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_invalid_hint:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4
+ ret void
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.mem_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.mem_hint", i32 1}
+!4 = !{i32 1, !5}
+!5 = !{!"aarch64.mem_hint", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
new file mode 100644
index 00000000000000..5f5bc430397782
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -0,0 +1,462 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
+
+;
+; W register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w2, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w2, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w2, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x2, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roW_mfloat(ptr %ptr, i32 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str s0, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str d0, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+;
+; X register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w2, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w2, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i64 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w2, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i64 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x2, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i64 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roX_mfloat(ptr %ptr, i64 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i64 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i64 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i64 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str s0, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i64 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str d0, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i64 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; Unsigned immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w1, [x0, #4095]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 4095
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 2048
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 1024
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 512
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_uimm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, #2048]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 2048
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 2048
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 2048
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str s0, [x0, #8192]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 2048
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str d0, [x0, #16384]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 2048
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+;
+; Signed immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturb w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 -256
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturh w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 -128
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 -64
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur x1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 -32
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_imm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturb w8, [x0, #-128]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 -128
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 -128
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 -128
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #512
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str s0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 -128
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sub x8, x0, #1024
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str d0, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 -128
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.mem_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.mem_hint", i32 1}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
new file mode 100644
index 00000000000000..31f82a33de8eb3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=prolog-epilog -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ define void @test_atomic_store_pseudo() nounwind { entry: unreachable }
+...
+---
+name: test_atomic_store_pseudo
+tracksRegLiveness: true
+liveins:
+ - { reg: '$w0' }
+ - { reg: '$s0' }
+ - { reg: '$x0' }
+ - { reg: '$d0' }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $w0, $s0, $x0, $d0
+
+ ; CHECK-LABEL: name: test_atomic_store_pseudo
+ ; CHECK: liveins: $w0, $s0, $x0, $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Bi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Hi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Wi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Si $s0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Xi $x0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Di $d0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Bui $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Hui $w0, $sp, 4, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Wui $w0, $sp, 2, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Sui $s0, $sp, 2, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Xui $x0, $sp, 1, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Dui $d0, $sp, 1, 1, 48
+ ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+ ; CHECK-NEXT: RET_ReallyLR
+ ATOMIC_STORE_HINT_Bi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Hi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Wi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Si $s0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Xi $x0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Di $d0, %stack.0, 0, 1, 48
+
+ ATOMIC_STORE_HINT_Bui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Hui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Wui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Sui $s0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Xui $x0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Dui $d0, %stack.0, 0, 1, 48
+
+ RET_ReallyLR
+...
More information about the llvm-commits
mailing list