[clang] [llvm] [AArch64] Implement the atomic store with hint intrinsic (PR #198316)
Kerry McLaughlin via cfe-commits
cfe-commits at lists.llvm.org
Wed Sep 2 07:11:43 PDT 2026
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/198316
>From 544a34d35a3877ffb161b1205721b5822abcc5f4 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 6 May 2026 13:08:02 +0000
Subject: [PATCH 1/8] [AArch64] Implement the atomic store with hint intrinsic
Adds the following ACLE intrinsic as described in [1]:
void __arm_atomic_store_with_hint(type *ptr, type data,
int memory_order, int hint);
A regular atomic store instruction is emitted in Clang for this builtin
with additional metadata (`!aarch64.atomic.hint`), which ensures the
instruction is recognised as atomic by passes in LLVM.
When an atomic store has this metadata, this lowers to the ATOMIC_STORE_HINT
pseudo which is later expanded by AArch64ExpandPseudoInsts into an STSHH
instruction plus an atomic store.
The hint value is represented using MOTargetFlag3 & MOTargetFlag4 flags,
which will need to be extended when new hints are added in future.
[1] https://github.com/ARM-software/acle/pull/432
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 +
.../clang/Basic/DiagnosticSemaKinds.td | 6 +
clang/include/clang/Sema/SemaARM.h | 1 +
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 54 ++++
clang/lib/Headers/arm_acle.h | 6 +
clang/lib/Sema/SemaARM.cpp | 92 ++++++
clang/test/CodeGen/arm_acle.c | 78 +++++
clang/test/CodeGen/builtins-arm64.c | 13 +
clang/test/Sema/builtins-arm64.c | 17 ++
.../include/llvm/Support/AArch64AtomicHints.h | 36 +++
llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 +
.../AArch64/AArch64ExpandPseudoInsts.cpp | 56 ++++
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 20 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 21 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 109 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 13 +
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 9 +
.../GISel/AArch64InstructionSelector.cpp | 61 ++++
.../Atomics/aarch64-atomic-store-hint.ll | 288 ++++++++++++++++++
.../Atomics/aarch64-atomic-store-hint.mir | 30 ++
20 files changed, 916 insertions(+), 1 deletion(-)
create mode 100644 llvm/include/llvm/Support/AArch64AtomicHints.h
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 4c49c874133fb..30aa3d526cbbb 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -171,6 +171,10 @@ let Attributes = [NoThrow], Features = "ls64" in {
def st64bv0 : AArch64TargetBuiltin<"uint64_t (void *, uint64_t const *)">;
}
+let Attributes = [NoThrow, CustomTypeChecking] in {
+ def atomic_store_with_hint : AArch64Builtin<"void(...)">;
+}
+
// Armv9.3-A Guarded Control Stack
let Attributes = [NoThrow], Features = "gcs" in {
def gcspopm : AArch64TargetBuiltin<"uint64_t (uint64_t)">;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index b5100b67dccd7..fd2badc29766d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9720,6 +9720,12 @@ def err_atomic_op_needs_atomic_int : Error<
def err_atomic_op_needs_atomic_fp
: Error<"address argument to atomic operation must be a pointer to "
"%select{|atomic }0floating point type (%1 invalid)">;
+def err_atomic_op_hint_data_size : Error<
+ "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
+def err_atomic_hint_has_invalid_memory_order : Error<
+ "invalid memory order argument to atomic hint operation (%0 invalid)">;
+def err_atomic_hint_has_invalid_hint_type : Error<
+ "invalid hint type argument to atomic hint operation (%0 invalid)">;
def warn_atomic_op_has_invalid_memory_order : Warning<
"%select{|success |failure }0memory order argument to atomic operation is invalid">,
InGroup<DiagGroup<"atomic-memory-ordering">>;
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index af8e0e9047171..b0a01c40ffece 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -70,6 +70,7 @@ class SemaARM : public SemaBase {
bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum,
unsigned ExpectedFieldNum, bool AllowName);
bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall);
bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 46f80979bcb97..32556d35fe117 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -20,6 +20,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include <numeric>
@@ -2065,6 +2066,56 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
Ops);
}
+static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
+ unsigned BuiltinID,
+ const CallExpr *E) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ CodeGen::CodeGenModule &CGM = CGF.CGM;
+ Expr::EvalResult Result;
+ if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer policy argument to atomic store with hint.");
+
+ StoreInst *Store =
+ Builder.CreateStore(CGF.EmitScalarExpr(E->getArg(1)), // Value
+ CGF.EmitPointerWithAlignment(E->getArg(0))); // Ptr;
+
+ AtomicOrdering Ordering;
+ unsigned OrderingArg = Result.Val.getInt().getExtValue();
+ assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
+
+ switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
+ default:
+ llvm_unreachable("Unsupported atomic ordering found.");
+ case AtomicOrderingCABI::relaxed:
+ Ordering = AtomicOrdering::Monotonic;
+ break;
+ case AtomicOrderingCABI::release:
+ Ordering = AtomicOrdering::Release;
+ break;
+ case AtomicOrderingCABI::seq_cst:
+ Ordering = AtomicOrdering::SequentiallyConsistent;
+ break;
+ }
+ Store->setAtomic(Ordering);
+
+ if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer hint argument to atomic store with hint.");
+ unsigned HintArg = Result.Val.getInt().getExtValue();
+ assert((getAtomicStoreHintFromMD(HintArg) !=
+ AArch64AtomicStoreHint::HINT_NONE) &&
+ "Invalid hint type");
+
+ MDNode *HintMDVal =
+ MDNode::get(CGM.getLLVMContext(),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
+ Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
+ HintMDVal);
+
+ return Store;
+}
+
/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
/// argument that specifies the vector type. The additional argument is meant
/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
@@ -4852,6 +4903,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
+
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 73b1e841b6719..8d4fbdc2e8c32 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -742,6 +742,12 @@ __arm_st64bv0(void *__addr, data512_t __value) {
}
#endif
+/* Atomic store with hints */
+#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
+#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
+#endif
+
/* 11.1 Special register intrinsics */
#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f38a35ec08fb0..7a03f25a7e5c9 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -17,6 +17,7 @@
#include "clang/Sema/Initialization.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
+#include "llvm/Support/AArch64AtomicHints.h"
namespace clang {
@@ -322,6 +323,94 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
return false;
}
+bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
+ CallExpr *TheCall) {
+ if (SemaRef.checkArgCount(TheCall, 4))
+ return true;
+
+ // Arg 0 should be the pointer type. The pointee type must be a
+ // scalar integral or floating-point type of 8, 16, 32 or 64 bits.
+ ASTContext &Context = getASTContext();
+ Expr *PtrArg = TheCall->getArg(0);
+ auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(PtrArg);
+ if (PtrArgRes.isInvalid())
+ return true;
+ auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
+ if (!PtrTy)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_builtin_must_be_pointer)
+ << PtrArg->getType() << 0 << PtrArg->getSourceRange();
+ QualType PtrQT = PtrTy->getPointeeType();
+
+ // TODO: Allow MFloat8 types when supported by atomic store
+ if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_atomic_int_or_fp)
+ << 0 << PtrQT << PtrArg->getSourceRange();
+
+ unsigned TySize =
+ Context.getTypeSize(Context.getCanonicalType(PtrQT).getUnqualifiedType());
+ if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
+ return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
+ << PtrArg->getSourceRange();
+
+ // Arg 1 is the data to be stored. The type must match the pointee
+ // type found above.
+ auto DataArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
+ if (DataArgRes.isInvalid())
+ return true;
+ QualType DataQT = DataArgRes.get()->getType();
+
+ if (PtrQT != DataQT)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_typecheck_call_different_arg_types)
+ << PtrQT << DataQT;
+
+ // Arg 2 is the memory order, which must be relaxed, release or seq_cst
+ auto MemOrdArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)).get();
+ std::optional<llvm::APSInt> MemOrdAP =
+ MemOrdArg->getIntegerConstantExpr(Context);
+ if (!MemOrdAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << MemOrdArg->getType() << MemOrdArg->getSourceRange();
+
+ unsigned Ordering = MemOrdAP->getZExtValue();
+ if (!llvm::isValidAtomicOrderingCABI(Ordering))
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+
+ auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
+ if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::release &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+
+ // Arg 3 is the hint type. Only values represented by AArch64AtomicStoreHint
+ // are valid.
+ auto HintArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
+ std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
+ if (!HintAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << HintArg->getType() << HintArg->getSourceRange();
+
+ unsigned Hint = HintAP->getZExtValue();
+ if (llvm::getAtomicStoreHintFromMD(Hint) ==
+ llvm::AArch64AtomicStoreHint::HINT_NONE)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << *HintAP << HintArg->getSourceRange();
+
+ return false;
+}
+
/// getNeonEltType - Return the QualType corresponding to the elements of
/// the vector type specified by the NeonTypeFlags. This is used to check
/// the pointer arguments for Neon load/store intrinsics.
@@ -1170,6 +1259,9 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
BuiltinID == AArch64::BI__builtin_arm_wsrp)
return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall);
+
// Only check the valid encoding range. Any constant in this range would be
// converted to a register of the form S2_2_C3_C4_5. Let the hardware throw
// an exception for incorrect registers. This matches MSVC behavior.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index e1bbc3d745bcd..29e35ed56fa69 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1761,3 +1761,81 @@ int test_rndrrs(uint64_t *__addr) {
return __rndrrs(__addr);
}
#endif
+
+#if defined(__ARM_64BIT_STATE)
+
+// AArch64-LABEL: @test_atomic_store_hint_char(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_char(char *ptr, char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_bfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_short(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_short(short *ptr, short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_u32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_float(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_float(float *ptr, float data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_s64(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_double(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_double(double *ptr, double data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+}
+
+// AArch64: [[META3]] = !{i32 0}
+// AArch64-NEXT: [[META4]] = !{i32 1}
+#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 3d054c79f1777..ad9ba7feca671 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -216,4 +216,17 @@ void trap() {
__builtin_arm_trap(42);
}
+void atomic_store_with_hint(int64_t *a, int64_t b) {
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !aarch64.atomic.hint ![[M1:[0-9]]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, 0);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !aarch64.atomic.hint ![[M1]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+}
+
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
+// CHECK: ![[M1]] = !{i32 0}
+// CHECK: ![[M2]] = !{i32 1}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 41cffd7ebb1a0..fb4718a1bd1f4 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -51,3 +51,20 @@ void test_trap(short s, unsigned short us) {
__builtin_arm_trap(s); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
__builtin_arm_trap(us); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
}
+
+void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
+ char c_data, __int128 inv_data, float f_data,
+ int inv_int) {
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+
+ __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
+ __builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-error {{invalid hint type argument to atomic hint operation (3 invalid)}}
+}
diff --git a/llvm/include/llvm/Support/AArch64AtomicHints.h b/llvm/include/llvm/Support/AArch64AtomicHints.h
new file mode 100644
index 0000000000000..8118f3e2df3ad
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64AtomicHints.h
@@ -0,0 +1,36 @@
+//===-- AArch64AtomicHints.h - AArch64 Atomic Hint Attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64ATOMICHINTS_H
+#define LLVM_SUPPORT_AARCH64ATOMICHINTS_H
+
+namespace llvm {
+enum class AArch64AtomicStoreHint {
+ HINT_NONE = 0,
+ HINT_STSHH_KEEP = 1,
+ HINT_STSHH_STRM = 2,
+};
+
+template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+ return (Int)AArch64AtomicStoreHint::HINT_STSHH_KEEP <= I &&
+ I <= (Int)AArch64AtomicStoreHint::HINT_STSHH_STRM;
+}
+
+template <typename Int>
+inline AArch64AtomicStoreHint getAtomicStoreHintFromMD(Int I) {
+ switch (I) {
+ case 0:
+ return AArch64AtomicStoreHint::HINT_STSHH_KEEP;
+ case 1:
+ return AArch64AtomicStoreHint::HINT_STSHH_STRM;
+ default:
+ return AArch64AtomicStoreHint::HINT_NONE;
+ }
+}
+} // namespace llvm
+#endif // LLVM_SUPPORT_AARCH64ATOMICHINTS_H
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index f2ffa40030cc0..ce12af884af32 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -255,6 +255,8 @@ static void copyMetadataForAtomic(Instruction &Dest,
Dest.setMetadata(ID, N);
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
+ else if (ID == Ctx.getMDKindID("aarch64.atomic.hint"))
+ Dest.setMetadata(ID, N);
// Losing amdgpu.ignore.denormal.mode, but it doesn't matter for current
// uses.
@@ -713,6 +715,7 @@ StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
Value *Addr = SI->getPointerOperand();
StoreInst *NewSI = Builder.CreateStore(NewVal, Addr, SI->getProperties());
+ copyMetadataForAtomic(*NewSI, *SI);
LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
SI->eraseFromParent();
return NewSI;
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 3965ff1783bc5..9193fd37f5318 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -68,6 +68,9 @@ class AArch64ExpandPseudoImpl {
MachineBasicBlock::iterator MBBI);
bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
+ bool expandAtomicStoreHintPseudo(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ unsigned Size);
bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
unsigned ExtendImm, unsigned ZeroReg,
@@ -1304,6 +1307,51 @@ bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
return true;
}
+bool AArch64ExpandPseudoImpl::expandAtomicStoreHintPseudo(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Size) {
+ MachineInstr &MI = *MBBI;
+ DebugLoc DL = MI.getDebugLoc();
+
+ unsigned StOpc;
+ unsigned Order = MI.getOperand(2).getImm();
+ bool Relaxed = Order == 0;
+
+ switch (Size) {
+ case 8:
+ StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
+ break;
+ case 16:
+ StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
+ break;
+ case 32:
+ StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
+ break;
+ case 64:
+ StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint size.");
+ }
+
+ auto *Hint = BuildMI(MBB, MBBI, DL, TII->get(AArch64::STSHH))
+ .addImm(MI.getOperand(3).getImm())
+ .getInstr();
+
+ auto Store = BuildMI(MBB, MBBI, DL, TII->get(StOpc))
+ .add(MI.getOperand(1))
+ .addReg(MI.getOperand(0).getReg())
+ .setMemRefs(MI.memoperands())
+ .setMIFlags(MI.getFlags());
+
+ if (Relaxed)
+ Store.addImm(0);
+
+ transferImpOps(MI, Store, Store);
+ finalizeBundle(MBB, Hint->getIterator(), MBBI->getIterator());
+ MI.eraseFromParent();
+ return true;
+}
+
/// If MBBI references a pseudo instruction that should be expanded here,
/// do the expansion and return true. Otherwise return false.
bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
@@ -2006,6 +2054,14 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case AArch64::NAND_ZZZ:
case AArch64::NOR_ZZZ:
return expandSVEBitwisePseudo(MI, MBB, MBBI);
+ case AArch64::ATOMIC_STORE_HINT_B:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 8);
+ case AArch64::ATOMIC_STORE_HINT_H:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 16);
+ case AArch64::ATOMIC_STORE_HINT_S:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 32);
+ case AArch64::ATOMIC_STORE_HINT_D:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 64);
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 531478517ef9a..b4ab4fd6d49fd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -12,6 +12,7 @@
#include "AArch64.h"
#include "AArch64ExpandImm.h"
+#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
@@ -516,6 +518,10 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
+ bool isAtomicHintInst(SDNode *N, AArch64AtomicStoreHint Hint) const;
+ bool isAtomicSTSHH_KEEP(SDNode *N) const;
+ bool isAtomicSTSHH_STRM(SDNode *N) const;
+
bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
bool Negate);
bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
@@ -4610,6 +4616,20 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
+bool AArch64DAGToDAGISel::isAtomicHintInst(SDNode *N,
+ AArch64AtomicStoreHint Hint) const {
+ const MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
+ return AArch64InstrInfo::decodeAtomicHintFlags(MMO->getFlags()) == Hint;
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
+ return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
+ return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+}
+
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
SDValue &Shift, bool Negate) {
if (!isa<ConstantSDNode>(N))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9f28ad1fdf660..8fc22d2283dad 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -79,6 +79,7 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
@@ -19431,7 +19432,25 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
I.hasMetadata(FALKOR_STRIDED_ACCESS_MD))
return MOStridedAccess;
- return MachineMemOperand::MONone;
+
+ auto Flags = MachineMemOperand::MONone;
+ const MDNode *AtomicStHint = I.getMetadata(AARCH64_ATOMIC_STORE_HINT_MD);
+ if (AtomicStHint) {
+ unsigned HintVal =
+ cast<ConstantInt>(
+ cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
+ ->getZExtValue();
+ AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
+ assert(Hint != AArch64AtomicStoreHint::HINT_NONE &&
+ "Unrecognised atomic hint value requested.");
+
+ if (static_cast<unsigned>(Hint) & 0b1)
+ Flags |= MOAtomicHintBit0;
+ if (static_cast<unsigned>(Hint) & 0b10)
+ Flags |= MOAtomicHintBit1;
+ }
+
+ return Flags;
}
bool AArch64TargetLowering::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2187f21abb70f..7d719239ecc02 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -282,6 +282,115 @@ def : Pat<(relaxed_store<atomic_store_64>
(am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
(STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
+//===----------------------------------
+// Atomic store with hint pseudos
+//===----------------------------------
+
+class seq_cst_store<PatFrags base>
+ : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
+ let IsAtomic = 1;
+ let IsAtomicOrderingSequentiallyConsistent = 1;
+}
+
+class atomic_hint_stshh_keep_relaxed<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (relaxed_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_keep_release<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (releasing_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_keep_seqcst<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (seq_cst_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_strm_relaxed<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (relaxed_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class atomic_hint_stshh_strm_release<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (releasing_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class atomic_hint_stshh_strm_seqcst<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (seq_cst_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class BaseStoreHintPseudo<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
+ let isCodeGenOnly = 1;
+ let hasSideEffects = 1;
+ let mayStore = 1;
+}
+
+def ATOMIC_STORE_HINT_B : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
+
+let AddedComplexity = 15 in {
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 1))>;
+
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 1))>;
+
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 1))>;
+}
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index fe0c59622e17b..0dd3e08033d52 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2954,6 +2954,19 @@ bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
});
}
+AArch64AtomicStoreHint
+AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
+ unsigned AtomicHint = 0;
+ if (MMOFlags & MOAtomicHintBit0)
+ AtomicHint += 0b1;
+ if (MMOFlags & MOAtomicHintBit1)
+ AtomicHint += 0b10;
+
+ if (!isValidAArch64AtomicHintValue(AtomicHint))
+ return AArch64AtomicStoreHint::HINT_NONE;
+ return static_cast<AArch64AtomicStoreHint>(AtomicHint);
+}
+
bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
switch (Opc) {
default:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index d9e34365479f1..fc5390f1b91ad 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -16,6 +16,7 @@
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/TypeSize.h"
#include <optional>
@@ -30,8 +31,13 @@ static const MachineMemOperand::Flags MOSuppressPair =
MachineMemOperand::MOTargetFlag1;
static const MachineMemOperand::Flags MOStridedAccess =
MachineMemOperand::MOTargetFlag2;
+static const MachineMemOperand::Flags MOAtomicHintBit0 =
+ MachineMemOperand::MOTargetFlag3;
+static const MachineMemOperand::Flags MOAtomicHintBit1 =
+ MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
+#define AARCH64_ATOMIC_STORE_HINT_MD "aarch64.atomic.hint"
// AArch64 MachineCombiner patterns
enum AArch64MachineCombinerPattern : unsigned {
@@ -230,6 +236,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
+ static AArch64AtomicStoreHint
+ decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags);
+
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
static bool hasUnscaledLdStOffset(MachineInstr &MI) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0b018dc7b22a8..fcfd223456cf4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -45,6 +45,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Type.h"
#include "llvm/Pass.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
@@ -2518,6 +2519,66 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
I.eraseFromParent();
return true;
}
+ case TargetOpcode::G_STORE: {
+ GStore &St = cast<GStore>(I);
+ auto MMO = St.getMMO();
+ LLT PtrTy = MRI.getType(St.getPointerReg());
+
+ // Only for handling atomic store with hint.
+ // Can only handle AddressSpace 0, 64-bit pointers.
+ if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64)) {
+ return false;
+ }
+
+ AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
+ if (Hint == AArch64AtomicStoreHint::HINT_NONE)
+ return false;
+
+ unsigned HintOpc;
+ unsigned StoreSize = St.getMemSizeInBits().getValue();
+ Register ValueReg = St.getValueReg();
+ switch (StoreSize) {
+ case 8:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_B;
+ break;
+ case 16: {
+ Register CastReg;
+ if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
+ auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
+ {&AArch64::FPR32RegClass}, {});
+ auto Ins = MIB.buildInstr(TargetOpcode::INSERT_SUBREG,
+ {&AArch64::FPR32RegClass}, {Undef, ValueReg})
+ .addImm(AArch64::hsub);
+ constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
+ constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
+ ValueReg = Ins.getReg(0);
+ }
+ HintOpc = AArch64::ATOMIC_STORE_HINT_H;
+ break;
+ }
+ case 32:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_S;
+ break;
+ case 64:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_D;
+ break;
+ default:
+ llvm_unreachable("Unexpected getMemSizeInBits() value for atomic hint.");
+ }
+
+ unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
+
+ auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
+ .addReg(St.getPointerReg())
+ .addReg(ValueReg)
+ .addImm((int)toCABI(St.getMMO().getSuccessOrdering()))
+ .addImm(static_cast<unsigned>(HintImm));
+
+ StrPseudo.cloneMemRefs(I);
+ I.eraseFromParent();
+ constrainSelectedInstRegOperands(*StrPseudo, TII, TRI, RBI);
+ return true;
+ }
default:
return false;
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
new file mode 100644
index 0000000000000..dfcfa92cbc6c8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -0,0 +1,288 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -verify-machineinstrs < %s | FileCheck %s
+
+;
+; STSHH: Keep, Relaxed
+;
+
+define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, Release
+;
+
+define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, SequentiallyConsistent
+;
+
+define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Stream, Relaxed
+;
+
+define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; STSHH: Stream, Release
+;
+
+define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; STSHH: Stream, SequentiallyConsistent
+;
+
+define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+!0 = !{i32 0}
+!1 = !{i32 1}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
new file mode 100644
index 0000000000000..ed69efafb04c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
@@ -0,0 +1,30 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ define void @test_atomic_store_keep_release_i8(ptr %ptr, i8 %val) {
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+ }
+
+ !0 = !{i32 0}
+...
+
+---
+name: test_atomic_store_keep_release_i8
+liveins:
+ - { reg: '$x0', virtual-reg: '' }
+ - { reg: '$w1', virtual-reg: '' }
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $w1, $x0
+
+ ; CHECK-LABEL: name: test_atomic_store_keep_release_i8
+ ; CHECK: BUNDLE implicit killed $w1, implicit $x0 :: (store release (s8) into %ir.ptr, align 8) {
+ ; CHECK-NEXT: STSHH 0
+ ; CHECK-NEXT: STRBBui killed renamable $w1, $x0, 0 :: (store release (s8) into %ir.ptr, align 8)
+ ; CHECK-NEXT: }
+ ; CHECK-NEXT: RET undef $lr
+
+ ATOMIC_STORE_HINT_B killed renamable $x0, killed renamable $w1, 0, 0 :: (store release (s8) into %ir.ptr, align 8)
+ RET_ReallyLR
+...
>From 133738679fb003f22287342b549d97c5b24dccb2 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 19 May 2026 10:43:46 +0000
Subject: [PATCH 2/8] Move expandAtomicStoreHintPseudo to AArch64AsmPrinter.cpp
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 51 +++++++++++++++++
.../AArch64/AArch64ExpandPseudoInsts.cpp | 56 -------------------
.../lib/Target/AArch64/AArch64InstrAtomics.td | 1 +
.../Atomics/aarch64-atomic-store-hint.mir | 30 ----------
4 files changed, 52 insertions(+), 86 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 005f69b08a4d3..dd34b4ccbc9d6 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -290,6 +290,9 @@ class AArch64AsmPrinter : public AsmPrinter {
// Emit expansion of Compare-and-branch pseudo instructions
void emitCBPseudoExpansion(const MachineInstr *MI);
+ // Emit expansion of atomic store with hint pseudo instructions
+ void emitAtomicHintPseudoExpansion(const MachineInstr *MI, unsigned Size);
+
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
EmitToStreamer(*OutStreamer, Inst);
@@ -3251,6 +3254,42 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, Inst);
}
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
+ unsigned Size) {
+
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(2).getImm();
+ bool Relaxed = Order == 0;
+ switch (Size) {
+ case 8:
+ StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
+ break;
+ case 16:
+ StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
+ break;
+ case 32:
+ StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
+ break;
+ case 64:
+ StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint size.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(3).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Store.setFlags(MI->getFlags());
+ if (Relaxed)
+ Store.addOperand(MCOperand::createImm(0));
+ EmitToStreamer(*OutStreamer, Store);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -3959,6 +3998,18 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::CBXPrr:
emitCBPseudoExpansion(MI);
return;
+ case AArch64::ATOMIC_STORE_HINT_B:
+ emitAtomicHintPseudoExpansion(MI, 8);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_H:
+ emitAtomicHintPseudoExpansion(MI, 16);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_S:
+ emitAtomicHintPseudoExpansion(MI, 32);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_D:
+ emitAtomicHintPseudoExpansion(MI, 64);
+ return;
}
if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 9193fd37f5318..3965ff1783bc5 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -68,9 +68,6 @@ class AArch64ExpandPseudoImpl {
MachineBasicBlock::iterator MBBI);
bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
- bool expandAtomicStoreHintPseudo(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- unsigned Size);
bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
unsigned ExtendImm, unsigned ZeroReg,
@@ -1307,51 +1304,6 @@ bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
return true;
}
-bool AArch64ExpandPseudoImpl::expandAtomicStoreHintPseudo(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Size) {
- MachineInstr &MI = *MBBI;
- DebugLoc DL = MI.getDebugLoc();
-
- unsigned StOpc;
- unsigned Order = MI.getOperand(2).getImm();
- bool Relaxed = Order == 0;
-
- switch (Size) {
- case 8:
- StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
- break;
- case 16:
- StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
- break;
- case 32:
- StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
- break;
- case 64:
- StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
- break;
- default:
- llvm_unreachable("Unexpected atomic hint size.");
- }
-
- auto *Hint = BuildMI(MBB, MBBI, DL, TII->get(AArch64::STSHH))
- .addImm(MI.getOperand(3).getImm())
- .getInstr();
-
- auto Store = BuildMI(MBB, MBBI, DL, TII->get(StOpc))
- .add(MI.getOperand(1))
- .addReg(MI.getOperand(0).getReg())
- .setMemRefs(MI.memoperands())
- .setMIFlags(MI.getFlags());
-
- if (Relaxed)
- Store.addImm(0);
-
- transferImpOps(MI, Store, Store);
- finalizeBundle(MBB, Hint->getIterator(), MBBI->getIterator());
- MI.eraseFromParent();
- return true;
-}
-
/// If MBBI references a pseudo instruction that should be expanded here,
/// do the expansion and return true. Otherwise return false.
bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
@@ -2054,14 +2006,6 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case AArch64::NAND_ZZZ:
case AArch64::NOR_ZZZ:
return expandSVEBitwisePseudo(MI, MBB, MBBI);
- case AArch64::ATOMIC_STORE_HINT_B:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 8);
- case AArch64::ATOMIC_STORE_HINT_H:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 16);
- case AArch64::ATOMIC_STORE_HINT_S:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 32);
- case AArch64::ATOMIC_STORE_HINT_D:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 64);
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 7d719239ecc02..af57d471e0411 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -325,6 +325,7 @@ class atomic_hint_stshh_strm_seqcst<PatFrag base>
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
+ let Size = 8;
let isCodeGenOnly = 1;
let hasSideEffects = 1;
let mayStore = 1;
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
deleted file mode 100644
index ed69efafb04c3..0000000000000
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
-
---- |
- define void @test_atomic_store_keep_release_i8(ptr %ptr, i8 %val) {
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
- ret void
- }
-
- !0 = !{i32 0}
-...
-
----
-name: test_atomic_store_keep_release_i8
-liveins:
- - { reg: '$x0', virtual-reg: '' }
- - { reg: '$w1', virtual-reg: '' }
-body: |
- bb.0 (%ir-block.0):
- liveins: $w1, $x0
-
- ; CHECK-LABEL: name: test_atomic_store_keep_release_i8
- ; CHECK: BUNDLE implicit killed $w1, implicit $x0 :: (store release (s8) into %ir.ptr, align 8) {
- ; CHECK-NEXT: STSHH 0
- ; CHECK-NEXT: STRBBui killed renamable $w1, $x0, 0 :: (store release (s8) into %ir.ptr, align 8)
- ; CHECK-NEXT: }
- ; CHECK-NEXT: RET undef $lr
-
- ATOMIC_STORE_HINT_B killed renamable $x0, killed renamable $w1, 0, 0 :: (store release (s8) into %ir.ptr, align 8)
- RET_ReallyLR
-...
>From 19f1620ddbd61468d453b48a788ec33257729c02 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 12 Jun 2026 10:38:00 +0000
Subject: [PATCH 3/8] - Document aarch64.atomic.hint in LangRef - Rewrite
patterns - Use LLVM ordering in patterns & expansion - Add Clang tests for
more types - Remove AARCH64_ATOMIC_STORE_HINT_MD
---
.../clang/Basic/DiagnosticSemaKinds.td | 10 +-
clang/lib/Sema/SemaARM.cpp | 9 +-
clang/test/CodeGen/arm_acle.c | 90 ++++++++++++++
clang/test/Sema/builtins-arm64.c | 6 +-
llvm/docs/LangRef.md | 26 ++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 115 ++++--------------
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 1 -
.../GISel/AArch64InstructionSelector.cpp | 32 ++++-
10 files changed, 185 insertions(+), 108 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index fd2badc29766d..5c7d696642583 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9652,6 +9652,11 @@ def err_atomic_builtin_must_be_pointer : Error<
def err_atomic_builtin_must_be_pointer_intptr : Error<
"address argument to atomic builtin must be a pointer to integer or pointer"
" (%0 invalid)">;
+
+def err_atomic_hint_builtin_must_be_pointer : Error<
+ "address argument to atomic hint builtin must be a pointer to a scalar "
+ "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+
def err_atomic_builtin_cannot_be_const : Error<
"address argument to atomic builtin cannot be const-qualified (%0 invalid)">;
def err_atomic_builtin_must_be_pointer_intfltptr : Error<
@@ -9724,8 +9729,9 @@ def err_atomic_op_hint_data_size : Error<
"address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
def err_atomic_hint_has_invalid_memory_order : Error<
"invalid memory order argument to atomic hint operation (%0 invalid)">;
-def err_atomic_hint_has_invalid_hint_type : Error<
- "invalid hint type argument to atomic hint operation (%0 invalid)">;
+def warn_atomic_hint_has_invalid_hint_type : Warning<
+ "unrecognised hint type argument to atomic hint operation (%0)">,
+ InGroup<DiagGroup<"atomic-hint-type">>;
def warn_atomic_op_has_invalid_memory_order : Warning<
"%select{|success |failure }0memory order argument to atomic operation is invalid">,
InGroup<DiagGroup<"atomic-memory-ordering">>;
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 7a03f25a7e5c9..c4da19526d0a7 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -338,7 +338,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_builtin_must_be_pointer)
+ diag::err_atomic_hint_builtin_must_be_pointer)
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
@@ -348,8 +348,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_op_needs_atomic_int_or_fp)
<< 0 << PtrQT << PtrArg->getSourceRange();
- unsigned TySize =
- Context.getTypeSize(Context.getCanonicalType(PtrQT).getUnqualifiedType());
+ unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
<< PtrArg->getSourceRange();
@@ -398,14 +397,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
if (!HintAP)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_hint_type)
+ diag::warn_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
unsigned Hint = HintAP->getZExtValue();
if (llvm::getAtomicStoreHintFromMD(Hint) ==
llvm::AArch64AtomicStoreHint::HINT_NONE)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_hint_type)
+ diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
return false;
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 29e35ed56fa69..1c42a4a171140 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1773,6 +1773,24 @@ void test_atomic_store_hint_char(char *ptr, char data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_uchar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_schar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_bfloat(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
@@ -1782,6 +1800,15 @@ void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
}
+// AArch64-LABEL: @test_atomic_store_hint_half(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_short(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
@@ -1791,6 +1818,33 @@ void test_atomic_store_hint_short(short *ptr, short data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_ushort(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_int(int *ptr, int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_unsigned(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_u32(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
@@ -1800,6 +1854,15 @@ void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_s32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_float(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
@@ -1818,6 +1881,24 @@ void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
}
+// AArch64-LABEL: @test_atomic_store_hint_long(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long(long *ptr, long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_ulong(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
@@ -1827,6 +1908,15 @@ void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_double(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index fb4718a1bd1f4..5372aff24b739 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -58,13 +58,13 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
- __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral or floating-point type of 8, 16, 32, or 64 bits ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
__builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int' invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-error {{invalid hint type argument to atomic hint operation (3 invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-warning {{unrecognised hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
}
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 26a5117708a76..9c56d35ad8b3b 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -8742,6 +8742,32 @@ to the SSA value of the pointer operand.
Note that this is an experimental feature, which means that its semantics might
change in the future.
+(metadata_aarch64_atomic_hint)=
+
+#### '``aarch64.atomic.hint``' Metadata
+
+The ``aarch64.atomic.hint`` metadata may be attached to an atomic store
+instruction, referencing a single metadata node containing a single ``i32``
+entry:
+
+```llvm
+ store atomic i64 %x, ptr %y seq_cst, align 8, !aarch64.atomic.hint !0
+
+ ...
+ !0 = !{i32 1}
+```
+
+On AArch64 targets, this metadata may be used to emit an atomic store together
+with a hint instruction. The hint is a suggestion to the compiler which may be
+used when selecting code sequences, but it is not required to emit a specific
+hint instruction. The following hint values are currently recognised:
+
+- ``0``: ``stshh keep`` hint.
+- ``1``: ``stshh strm`` hint.
+
+If the compiler does not recognise the hint value provided, it may ignore the
+metadata. Targets that do not support this metadata may also ignore it.
+
#### '`type`' Metadata
See {doc}`TypeMetadata`.
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index dd34b4ccbc9d6..d967d3df4e3fd 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3259,7 +3259,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
unsigned StOpc;
unsigned Order = MI->getOperand(2).getImm();
- bool Relaxed = Order == 0;
+ bool Relaxed = Order == 2;
switch (Size) {
case 8:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8fc22d2283dad..439402f102737 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19434,7 +19434,7 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
return MOStridedAccess;
auto Flags = MachineMemOperand::MONone;
- const MDNode *AtomicStHint = I.getMetadata(AARCH64_ATOMIC_STORE_HINT_MD);
+ const MDNode *AtomicStHint = I.getMetadata("aarch64.atomic.hint");
if (AtomicStHint) {
unsigned HintVal =
cast<ConstantInt>(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index af57d471e0411..af06ef9014031 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -286,42 +286,6 @@ def : Pat<(relaxed_store<atomic_store_64>
// Atomic store with hint pseudos
//===----------------------------------
-class seq_cst_store<PatFrags base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
- let IsAtomic = 1;
- let IsAtomicOrderingSequentiallyConsistent = 1;
-}
-
-class atomic_hint_stshh_keep_relaxed<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (relaxed_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_keep_release<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (releasing_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_keep_seqcst<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (seq_cst_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_strm_relaxed<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (relaxed_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
-class atomic_hint_stshh_strm_release<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (releasing_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
-class atomic_hint_stshh_strm_seqcst<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (seq_cst_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
@@ -336,60 +300,33 @@ def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
+class atomic_hint_store<PatFrag Base, bit Rel, bit SeqCst, code Pred, code GIPred>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (Base node:$val, node:$ptr), Pred> {
+ let IsAtomic = 1;
+ let IsAtomicOrderingReleaseOrStronger = Rel;
+ let IsAtomicOrderingSequentiallyConsistent = SeqCst;
+ let GISelPredicateCode = GIPred;
+}
+
+multiclass AtomicHintPatterns<int Order, int Hint, bit Rel, bit SeqCst, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
+}
+
let AddedComplexity = 15 in {
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 1))>;
-
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 1))>;
-
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 1))>;
+ defm : AtomicHintPatterns<2, 0, 0, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 0, 1, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 0, 0, 1, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 1, 0, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 1, 1, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 1, 0, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index fc5390f1b91ad..57bf875dd2a7f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -37,7 +37,6 @@ static const MachineMemOperand::Flags MOAtomicHintBit1 =
MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
-#define AARCH64_ATOMIC_STORE_HINT_MD "aarch64.atomic.hint"
// AArch64 MachineCombiner patterns
enum AArch64MachineCombinerPattern : unsigned {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index fcfd223456cf4..e717d2c36d746 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -527,6 +527,11 @@ class AArch64InstructionSelector : public InstructionSelector {
MachineOperand &Predicate,
MachineIRBuilder &MIRBuilder) const;
+ bool isAtomicHintInst(const MachineInstr &MI,
+ AArch64AtomicStoreHint Hint) const;
+ bool isAtomicSTSHH_KEEP(const MachineInstr &MI) const;
+ bool isAtomicSTSHH_STRM(const MachineInstr &MI) const;
+
/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
@@ -2523,17 +2528,15 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
GStore &St = cast<GStore>(I);
auto MMO = St.getMMO();
LLT PtrTy = MRI.getType(St.getPointerReg());
+ AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
// Only for handling atomic store with hint.
// Can only handle AddressSpace 0, 64-bit pointers.
- if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64)) {
+ if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64) ||
+ Hint == AArch64AtomicStoreHint::HINT_NONE) {
return false;
}
- AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
- if (Hint == AArch64AtomicStoreHint::HINT_NONE)
- return false;
-
unsigned HintOpc;
unsigned StoreSize = St.getMemSizeInBits().getValue();
Register ValueReg = St.getValueReg();
@@ -2571,7 +2574,7 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
.addReg(St.getPointerReg())
.addReg(ValueReg)
- .addImm((int)toCABI(St.getMMO().getSuccessOrdering()))
+ .addImm((int)MMO.getSuccessOrdering())
.addImm(static_cast<unsigned>(HintImm));
StrPseudo.cloneMemRefs(I);
@@ -8208,6 +8211,23 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
.getZExtValue()));
}
+bool AArch64InstructionSelector::isAtomicHintInst(
+ const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
+ const GStore &St = cast<GStore>(MI);
+ auto MMO = St.getMMO();
+ return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
+}
+
+bool AArch64InstructionSelector::isAtomicSTSHH_KEEP(
+ const MachineInstr &MI) const {
+ return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+}
+
+bool AArch64InstructionSelector::isAtomicSTSHH_STRM(
+ const MachineInstr &MI) const {
+ return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+}
+
bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
>From 6d9b7fa4037d888afbe1afdf45fa6aab5cc9a080 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 23 Jun 2026 10:28:36 +0000
Subject: [PATCH 4/8] - Add support for different addressing modes when the
ordering is relaxed - Set IsAtomicOrderingMonotonic & IsAtomicOrderingRelease
in atomic_hint_store - Do not attach invalid hints to atomic_store & ignore
any invalid hints found
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 18 +-
clang/lib/Sema/SemaARM.cpp | 19 +-
clang/test/CodeGen/builtins-arm64.c | 4 +
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 143 +++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 -
.../lib/Target/AArch64/AArch64InstrAtomics.td | 137 ++++--
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 -
.../GISel/AArch64InstructionSelector.cpp | 92 ++--
.../Atomics/aarch64-atomic-store-hint.ll | 62 ++-
.../Atomics/aarch64-relaxed-store-hint.ll | 416 ++++++++++++++++++
10 files changed, 780 insertions(+), 115 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 32556d35fe117..4a9aecd13d8ad 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2103,15 +2103,15 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
llvm_unreachable(
"Expected integer hint argument to atomic store with hint.");
unsigned HintArg = Result.Val.getInt().getExtValue();
- assert((getAtomicStoreHintFromMD(HintArg) !=
- AArch64AtomicStoreHint::HINT_NONE) &&
- "Invalid hint type");
-
- MDNode *HintMDVal =
- MDNode::get(CGM.getLLVMContext(),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
- Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
- HintMDVal);
+
+ // Attach the hint if valid
+ if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
+ MDNode *HintMDVal =
+ MDNode::get(CGM.getLLVMContext(),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
+ Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
+ HintMDVal);
+ }
return Store;
}
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index c4da19526d0a7..95f4b4237f1e8 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -331,10 +331,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// Arg 0 should be the pointer type. The pointee type must be a
// scalar integral or floating-point type of 8, 16, 32 or 64 bits.
ASTContext &Context = getASTContext();
- Expr *PtrArg = TheCall->getArg(0);
- auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(PtrArg);
+ auto PtrArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
if (PtrArgRes.isInvalid())
return true;
+ auto *PtrArg = PtrArgRes.get();
auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
@@ -395,17 +396,17 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP)
- return Diag(TheCall->getBeginLoc(),
- diag::warn_atomic_hint_has_invalid_hint_type)
- << HintArg->getType() << HintArg->getSourceRange();
+ if (!HintAP) {
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << HintArg->getType() << HintArg->getSourceRange();
+ return false;
+ }
unsigned Hint = HintAP->getZExtValue();
if (llvm::getAtomicStoreHintFromMD(Hint) ==
llvm::AArch64AtomicStoreHint::HINT_NONE)
- return Diag(TheCall->getBeginLoc(),
- diag::warn_atomic_hint_has_invalid_hint_type)
- << *HintAP << HintArg->getSourceRange();
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << *HintAP << HintArg->getSourceRange();
return false;
}
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index ad9ba7feca671..01332f8114a11 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -225,6 +225,10 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+
+ // Invalid hint should be dropped
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index d967d3df4e3fd..9aac76885b438 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -291,7 +291,9 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitCBPseudoExpansion(const MachineInstr *MI);
// Emit expansion of atomic store with hint pseudo instructions
- void emitAtomicHintPseudoExpansion(const MachineInstr *MI, unsigned Size);
+ void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
@@ -3254,23 +3256,22 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, Inst);
}
-void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
- unsigned Size) {
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
unsigned StOpc;
unsigned Order = MI->getOperand(2).getImm();
bool Relaxed = Order == 2;
- switch (Size) {
- case 8:
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_B:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
break;
- case 16:
+ case AArch64::ATOMIC_STORE_HINT_H:
StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
break;
- case 32:
+ case AArch64::ATOMIC_STORE_HINT_S:
StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
break;
- case 64:
+ case AArch64::ATOMIC_STORE_HINT_D:
StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
break;
default:
@@ -3290,6 +3291,104 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
EmitToStreamer(*OutStreamer, Store);
}
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(5).getImm();
+ assert(Order == 2 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ StOpc = AArch64::STRBBroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ StOpc = AArch64::STRHHroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ StOpc = AArch64::STRWroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ StOpc = AArch64::STRXroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ StOpc = AArch64::STRBBroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ StOpc = AArch64::STRHHroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ StOpc = AArch64::STRWroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ StOpc = AArch64::STRXroX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(6).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm
+ Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed
+ Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(3).getImm();
+ assert(Order == 2 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ StOpc = AArch64::STRBBui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ StOpc = AArch64::STRHHui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ StOpc = AArch64::STRWui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ StOpc = AArch64::STRXui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ StOpc = AArch64::STURBBi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ StOpc = AArch64::STURHHi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ StOpc = AArch64::STURWi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ StOpc = AArch64::STURXi;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(4).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -3999,16 +4098,30 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
emitCBPseudoExpansion(MI);
return;
case AArch64::ATOMIC_STORE_HINT_B:
- emitAtomicHintPseudoExpansion(MI, 8);
- return;
case AArch64::ATOMIC_STORE_HINT_H:
- emitAtomicHintPseudoExpansion(MI, 16);
- return;
case AArch64::ATOMIC_STORE_HINT_S:
- emitAtomicHintPseudoExpansion(MI, 32);
- return;
case AArch64::ATOMIC_STORE_HINT_D:
- emitAtomicHintPseudoExpansion(MI, 64);
+ emitAtomicHintPseudoExpansion(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ emitAtomicHintPseudoExpansionRO(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ emitAtomicHintPseudoExpansionImm(MI);
return;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 439402f102737..078db000d79db 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19441,8 +19441,6 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
->getZExtValue();
AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
- assert(Hint != AArch64AtomicStoreHint::HINT_NONE &&
- "Unrecognised atomic hint value requested.");
if (static_cast<unsigned>(Hint) & 0b1)
Flags |= MOAtomicHintBit0;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index af06ef9014031..e0dc12bc62de6 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -286,47 +286,126 @@ def : Pat<(relaxed_store<atomic_store_64>
// Atomic store with hint pseudos
//===----------------------------------
-class BaseStoreHintPseudo<RegisterClass regtype>
- : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
- let Size = 8;
- let isCodeGenOnly = 1;
- let hasSideEffects = 1;
- let mayStore = 1;
-}
-
-def ATOMIC_STORE_HINT_B : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
-
-class atomic_hint_store<PatFrag Base, bit Rel, bit SeqCst, code Pred, code GIPred>
- : PatFrag<(ops node:$ptr, node:$val),
+let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
+ class BaseStoreHintPseudo<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, GPR32:$Rm, regtype:$data, WExt:$extend,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, GPR64:$Rm, regtype:$data, XExt:$extend,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, UImmOp:$offset,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoImm<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, simm9:$offset,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+}
+
+multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> {
+ def NAME : BaseStoreHintPseudo<regtype>;
+ def NAME # roW : BaseStoreHintPseudoROW<regtype, WExt>;
+ def NAME # roX : BaseStoreHintPseudoROX<regtype, XExt>;
+ def NAME # ui : BaseStoreHintPseudoUImm<regtype, UImmOp>;
+ def NAME # i : BaseStoreHintPseudoImm<regtype>;
+}
+
+defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>;
+defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
+defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+
+class atomic_hint_store<PatFrag Base, int Order, code Pred, code GIPred>
+ : PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
- let IsAtomicOrderingReleaseOrStronger = Rel;
- let IsAtomicOrderingSequentiallyConsistent = SeqCst;
+ let IsAtomicOrderingMonotonic = !eq(Order, 2);
+ let IsAtomicOrderingRelease = !eq(Order, 5);
+ let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
let GISelPredicateCode = GIPred;
}
-multiclass AtomicHintPatterns<int Order, int Hint, bit Rel, bit SeqCst, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+multiclass AtomicHintPatternsAddrMode<int Hint, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
+ (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
+ (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+}
+
+multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR64:$data),
+ def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred, GIPred> GPR64:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatterns<2, 0, 0, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 0, 1, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 0, 0, 1, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 1, 0, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 1, 1, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 1, 0, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+
+ defm : AtomicHintPatterns<2, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 0dd3e08033d52..25e0730149366 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2962,8 +2962,6 @@ AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
if (MMOFlags & MOAtomicHintBit1)
AtomicHint += 0b10;
- if (!isValidAArch64AtomicHintValue(AtomicHint))
- return AArch64AtomicStoreHint::HINT_NONE;
return static_cast<AArch64AtomicStoreHint>(AtomicHint);
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index e717d2c36d746..09df59d82cc85 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2537,14 +2537,11 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
return false;
}
- unsigned HintOpc;
- unsigned StoreSize = St.getMemSizeInBits().getValue();
+ // AtomicExpandPass converts FP stores to integer stores of the equivalent
+ // bitwidth. Widen the register here if the original type was f16.
+ unsigned StoreSize = St.getMemSize().getValue();
Register ValueReg = St.getValueReg();
- switch (StoreSize) {
- case 8:
- HintOpc = AArch64::ATOMIC_STORE_HINT_B;
- break;
- case 16: {
+ if (StoreSize == 2) {
Register CastReg;
if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
@@ -2556,26 +2553,72 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
ValueReg = Ins.getReg(0);
}
- HintOpc = AArch64::ATOMIC_STORE_HINT_H;
- break;
}
- case 32:
- HintOpc = AArch64::ATOMIC_STORE_HINT_S;
- break;
- case 64:
- HintOpc = AArch64::ATOMIC_STORE_HINT_D;
- break;
- default:
- llvm_unreachable("Unexpected getMemSizeInBits() value for atomic hint.");
+
+ static constexpr unsigned BaseOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_B, AArch64::ATOMIC_STORE_HINT_H,
+ AArch64::ATOMIC_STORE_HINT_S, AArch64::ATOMIC_STORE_HINT_D};
+ static constexpr unsigned RegWOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_BroW, AArch64::ATOMIC_STORE_HINT_HroW,
+ AArch64::ATOMIC_STORE_HINT_SroW, AArch64::ATOMIC_STORE_HINT_DroW};
+ static constexpr unsigned RegXOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_BroX, AArch64::ATOMIC_STORE_HINT_HroX,
+ AArch64::ATOMIC_STORE_HINT_SroX, AArch64::ATOMIC_STORE_HINT_DroX};
+ static constexpr unsigned UImmOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_Bui, AArch64::ATOMIC_STORE_HINT_Hui,
+ AArch64::ATOMIC_STORE_HINT_Sui, AArch64::ATOMIC_STORE_HINT_Dui};
+ static constexpr unsigned ImmOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_Bi, AArch64::ATOMIC_STORE_HINT_Hi,
+ AArch64::ATOMIC_STORE_HINT_Si, AArch64::ATOMIC_STORE_HINT_Di};
+
+ AtomicOrdering Ordering = MMO.getSuccessOrdering();
+ MachineInstrBuilder StrPseudo;
+ unsigned HintOpc = 0;
+
+ // Other addressing modes can be used when the ordering is monotonic.
+ // Try to match these first, before falling back on the basic operands.
+ auto AddModeWRO = selectAddrModeWRO(St.getOperand(1), StoreSize);
+ auto AddModeXRO = selectAddrModeXRO(St.getOperand(1), StoreSize);
+ auto AddModeUImm = selectAddrModeIndexed(St.getOperand(1), StoreSize);
+ auto AddModeImm = selectAddrModeUnscaled(St.getOperand(1), StoreSize);
+
+ if (AddModeWRO && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = RegWOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeWRO)[0](StrPseudo);
+ (*AddModeWRO)[1](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeWRO)[2](StrPseudo);
+ } else if (AddModeXRO && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = RegXOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeXRO)[0](StrPseudo);
+ (*AddModeXRO)[1](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeXRO)[2](StrPseudo);
+ } else if (AddModeUImm && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = UImmOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeUImm)[0](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeUImm)[1](StrPseudo);
+ } else if (AddModeImm && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = ImmOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeImm)[0](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeImm)[1](StrPseudo);
+ } else {
+ HintOpc = BaseOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
+ .addReg(St.getPointerReg())
+ .addReg(ValueReg);
}
+ // Add the ordering and hint operands, before erasing the store.
unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
-
- auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
- .addReg(St.getPointerReg())
- .addReg(ValueReg)
- .addImm((int)MMO.getSuccessOrdering())
- .addImm(static_cast<unsigned>(HintImm));
+ StrPseudo.addImm((int)Ordering);
+ StrPseudo.addImm(HintImm);
StrPseudo.cloneMemRefs(I);
I.eraseFromParent();
@@ -8213,8 +8256,7 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
bool AArch64InstructionSelector::isAtomicHintInst(
const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
- const GStore &St = cast<GStore>(MI);
- auto MMO = St.getMMO();
+ auto MMO = cast<GStore>(MI).getMMO();
return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index dfcfa92cbc6c8..21aec7dae322f 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -6,7 +6,7 @@
; STSHH: Keep, Relaxed
;
-define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -16,7 +16,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) noun
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -26,7 +26,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) no
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -36,7 +36,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) no
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -50,7 +50,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) no
; STSHH: Keep, Release
;
-define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -62,7 +62,7 @@ define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %v
ret void
}
-define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_half:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -74,7 +74,7 @@ define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_float:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -85,7 +85,7 @@ define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val
ret void
}
-define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -100,7 +100,7 @@ define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %v
; STSHH: Keep, SequentiallyConsistent
;
-define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -110,7 +110,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounw
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -120,7 +120,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nou
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -130,7 +130,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nou
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -144,7 +144,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nou
; STSHH: Stream, Relaxed
;
-define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -156,7 +156,7 @@ define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %v
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -168,7 +168,7 @@ define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -179,7 +179,7 @@ define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -194,7 +194,7 @@ define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %v
; STSHH: Stream, Release
;
-define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -204,7 +204,7 @@ define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) no
ret void
}
-define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -214,7 +214,7 @@ define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val)
ret void
}
-define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -224,7 +224,7 @@ define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val)
ret void
}
-define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -238,7 +238,7 @@ define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val)
; STSHH: Stream, SequentiallyConsistent
;
-define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -250,7 +250,7 @@ define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -262,7 +262,7 @@ define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -273,7 +273,7 @@ define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val)
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -284,5 +284,19 @@ define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %
ret void
}
+;
+; Invalid Hint
+;
+
+define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_invalid_hint:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !2
+ ret void
+}
+
!0 = !{i32 0}
!1 = !{i32 1}
+!2 = !{i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
new file mode 100644
index 0000000000000..d80d9460c107c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -0,0 +1,416 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -global-isel=1 < %s | FileCheck %s
+
+;
+; W register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w2, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w2, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w2, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x2, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; X register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w2, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w2, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i64 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w2, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i64 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x2, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i64 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i64 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i64 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w8, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i64 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x8, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i64 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; Unsigned immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w1, [x0, #4095]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 4095
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 2048
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 1024
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 512
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 2048
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 2048
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w8, [x0, #8192]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 2048
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x8, [x0, #16384]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 2048
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; Signed immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturb w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 -256
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturh w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 -128
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 -64
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur x1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 -32
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 -128
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 -128
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: sub x8, x0, #512
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w9, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 -128
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x9, d0
+; CHECK-NEXT: sub x8, x0, #1024
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x9, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 -128
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+!0 = !{ i32 0 }
+!1 = !{ i32 1 }
>From 1f05fe7654c656f00a45a1f3f270e3e35dba3f13 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 12:55:03 +0000
Subject: [PATCH 5/8] - Replace STSHH with HINT after rebase
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 6 +++---
llvm/lib/Target/AArch64/AArch64InstrAtomics.td | 18 +++++++++---------
.../GISel/AArch64InstructionSelector.cpp | 2 +-
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 9aac76885b438..c7277637f3cb5 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3279,7 +3279,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(3).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(3).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
@@ -3328,7 +3328,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(6).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(6).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
@@ -3378,7 +3378,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(4).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index e0dc12bc62de6..8ab0c13e41edc 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -397,15 +397,15 @@ multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatternsAddrMode<0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatternsAddrMode<1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
-
- defm : AtomicHintPatterns<2, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+
+ defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 09df59d82cc85..0f39bb84122da 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2616,7 +2616,7 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
}
// Add the ordering and hint operands, before erasing the store.
- unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
+ unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 48 : 49;
StrPseudo.addImm((int)Ordering);
StrPseudo.addImm(HintImm);
>From 0ae2cb6a2e7ea7910f5cfbf233e3458b525a4198 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 13:26:06 +0000
Subject: [PATCH 6/8] - Remove aarch64.atomic.hint & use mem.cache_hint -
Remove GISel support for now and added atomic stores with the hint to
fallBackToDAGISel - Add mfp8 support in Clang + tests
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 14 +-
clang/lib/Sema/SemaARM.cpp | 4 +-
clang/test/CodeGen/arm_acle.c | 52 +++++---
clang/test/CodeGen/builtins-arm64.c | 12 +-
llvm/docs/LangRef.md | 26 ----
llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 4 +-
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 ++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 28 ++--
.../lib/Target/AArch64/AArch64InstrAtomics.td | 68 +++++-----
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 12 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 7 +-
.../GISel/AArch64InstructionSelector.cpp | 122 -----------------
.../Atomics/aarch64-atomic-store-hint.ll | 97 ++++++++++----
.../Atomics/aarch64-relaxed-store-hint.ll | 124 +++++++++++++-----
15 files changed, 282 insertions(+), 323 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 4a9aecd13d8ad..8ab9ecb65b4c4 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2106,11 +2106,15 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
// Attach the hint if valid
if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
- MDNode *HintMDVal =
- MDNode::get(CGM.getLLVMContext(),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
- Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
- HintMDVal);
+ LLVMContext &Ctx = CGM.getLLVMContext();
+ MDNode *AtomicHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.atomic_hint"),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
+ MDNode *HintNode = MDNode::get(
+ CGM.getLLVMContext(),
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), AtomicHint});
+
+ Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
}
return Store;
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 95f4b4237f1e8..b3c139bf1c8bc 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -343,8 +343,8 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
- // TODO: Allow MFloat8 types when supported by atomic store
- if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType())
+ if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType() &&
+ !PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_op_needs_atomic_int_or_fp)
<< 0 << PtrQT << PtrArg->getSourceRange();
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 1c42a4a171140..041530a5d2269 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1766,7 +1766,7 @@ int test_rndrrs(uint64_t *__addr) {
// AArch64-LABEL: @test_atomic_store_hint_char(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3:![0-9]+]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1:![0-9]+]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_char(char *ptr, char data) {
@@ -1775,7 +1775,7 @@ void test_atomic_store_hint_char(char *ptr, char data) {
// AArch64-LABEL: @test_atomic_store_hint_uchar(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
@@ -1784,16 +1784,25 @@ void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
// AArch64-LABEL: @test_atomic_store_hint_schar(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_mfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic <1 x i8> [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_mfloat(__mfp8 *ptr, __mfp8 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_bfloat(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3:![0-9]+]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
@@ -1802,7 +1811,7 @@ void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
// AArch64-LABEL: @test_atomic_store_hint_half(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
@@ -1811,7 +1820,7 @@ void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
// AArch64-LABEL: @test_atomic_store_hint_short(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_short(short *ptr, short data) {
@@ -1820,7 +1829,7 @@ void test_atomic_store_hint_short(short *ptr, short data) {
// AArch64-LABEL: @test_atomic_store_hint_ushort(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
@@ -1829,7 +1838,7 @@ void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
// AArch64-LABEL: @test_atomic_store_hint_int(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_int(int *ptr, int data) {
@@ -1838,7 +1847,7 @@ void test_atomic_store_hint_int(int *ptr, int data) {
// AArch64-LABEL: @test_atomic_store_hint_unsigned(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
@@ -1847,7 +1856,7 @@ void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
// AArch64-LABEL: @test_atomic_store_hint_u32(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
@@ -1856,7 +1865,7 @@ void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
// AArch64-LABEL: @test_atomic_store_hint_s32(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
@@ -1865,7 +1874,7 @@ void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
// AArch64-LABEL: @test_atomic_store_hint_float(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_float(float *ptr, float data) {
@@ -1874,7 +1883,7 @@ void test_atomic_store_hint_float(float *ptr, float data) {
// AArch64-LABEL: @test_atomic_store_hint_s64(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
@@ -1883,7 +1892,7 @@ void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
// AArch64-LABEL: @test_atomic_store_hint_long(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long(long *ptr, long data) {
@@ -1892,7 +1901,7 @@ void test_atomic_store_hint_long(long *ptr, long data) {
// AArch64-LABEL: @test_atomic_store_hint_ulong(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
@@ -1901,7 +1910,7 @@ void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
@@ -1910,7 +1919,7 @@ void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data
// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
@@ -1919,13 +1928,16 @@ void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned
// AArch64-LABEL: @test_atomic_store_hint_double(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_double(double *ptr, double data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
}
-// AArch64: [[META3]] = !{i32 0}
-// AArch64-NEXT: [[META4]] = !{i32 1}
+// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
+// AArch64-NEXT: [[HINT2]] = !{!"aarch64.atomic_hint", i32 0}
+
+// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
+// AArch64-NEXT: [[HINT4]] = !{!"aarch64.atomic_hint", i32 1}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 01332f8114a11..ed2bb6e65b525 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -218,13 +218,13 @@ void trap() {
void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !aarch64.atomic.hint ![[M1:[0-9]]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M1:[0-9]]]
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, 0);
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !aarch64.atomic.hint ![[M1]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M1]]
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
@@ -232,5 +232,7 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
-// CHECK: ![[M1]] = !{i32 0}
-// CHECK: ![[M2]] = !{i32 1}
+// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
+// CHECK: ![[M2]] = !{!"aarch64.atomic_hint", i32 0}
+// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
+// CHECK: ![[M4]] = !{!"aarch64.atomic_hint", i32 1}
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 9c56d35ad8b3b..26a5117708a76 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -8742,32 +8742,6 @@ to the SSA value of the pointer operand.
Note that this is an experimental feature, which means that its semantics might
change in the future.
-(metadata_aarch64_atomic_hint)=
-
-#### '``aarch64.atomic.hint``' Metadata
-
-The ``aarch64.atomic.hint`` metadata may be attached to an atomic store
-instruction, referencing a single metadata node containing a single ``i32``
-entry:
-
-```llvm
- store atomic i64 %x, ptr %y seq_cst, align 8, !aarch64.atomic.hint !0
-
- ...
- !0 = !{i32 1}
-```
-
-On AArch64 targets, this metadata may be used to emit an atomic store together
-with a hint instruction. The hint is a suggestion to the compiler which may be
-used when selecting code sequences, but it is not required to emit a specific
-hint instruction. The following hint values are currently recognised:
-
-- ``0``: ``stshh keep`` hint.
-- ``1``: ``stshh strm`` hint.
-
-If the compiler does not recognise the hint value provided, it may ignore the
-metadata. Targets that do not support this metadata may also ignore it.
-
#### '`type`' Metadata
See {doc}`TypeMetadata`.
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index ce12af884af32..9079c5ce50efe 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -244,6 +244,7 @@ static void copyMetadataForAtomic(Instruction &Dest,
case LLVMContext::MD_tbaa:
case LLVMContext::MD_tbaa_struct:
case LLVMContext::MD_alias_scope:
+ case LLVMContext::MD_mem_cache_hint:
case LLVMContext::MD_noalias:
case LLVMContext::MD_noalias_addrspace:
case LLVMContext::MD_access_group:
@@ -255,8 +256,6 @@ static void copyMetadataForAtomic(Instruction &Dest,
Dest.setMetadata(ID, N);
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
- else if (ID == Ctx.getMDKindID("aarch64.atomic.hint"))
- Dest.setMetadata(ID, N);
// Losing amdgpu.ignore.denormal.mode, but it doesn't matter for current
// uses.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 710a1eda00cc1..9a90ec499b8f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5417,10 +5417,10 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
report_fatal_error("Cannot generate unaligned atomic store");
auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
-
- MachineFunction &MF = DAG.getMachineFunction();
const MDNode *MemCacheHint =
getMemCacheHintMetadata(I, I.getPointerOperandIndex());
+
+ MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index b4ab4fd6d49fd..bd00fdceb8bab 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -518,7 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
- bool isAtomicHintInst(SDNode *N, AArch64AtomicStoreHint Hint) const;
+ AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand *MMO) const;
bool isAtomicSTSHH_KEEP(SDNode *N) const;
bool isAtomicSTSHH_STRM(SDNode *N) const;
@@ -4616,18 +4616,36 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
-bool AArch64DAGToDAGISel::isAtomicHintInst(SDNode *N,
- AArch64AtomicStoreHint Hint) const {
- const MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
- return AArch64InstrInfo::decodeAtomicHintFlags(MMO->getFlags()) == Hint;
+AArch64AtomicStoreHint
+AArch64DAGToDAGISel::decodeAtomicHintFlags(MachineMemOperand *MMO) const {
+ int AtomicHint = -1;
+ const MDNode *MemCacheHint = MMO->getMemCacheHint();
+ if (!MemCacheHint)
+ return AArch64AtomicStoreHint::HINT_NONE;
+
+ for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
+ const auto *Key = cast<MDString>(MemCacheHint->getOperand(I));
+ StringRef KeyStr = Key->getString();
+ const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
+
+ if (KeyStr != "aarch64.atomic_hint")
+ continue;
+
+ AtomicHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
+ ->getZExtValue();
+ }
+
+ return getAtomicStoreHintFromMD(AtomicHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
- return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+ return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64AtomicStoreHint::HINT_STSHH_KEEP;
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
- return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+ return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64AtomicStoreHint::HINT_STSHH_STRM;
}
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 078db000d79db..f29d68e7ae526 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -41,6 +41,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
@@ -79,7 +80,6 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
-#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
@@ -19432,23 +19432,7 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
I.hasMetadata(FALKOR_STRIDED_ACCESS_MD))
return MOStridedAccess;
-
- auto Flags = MachineMemOperand::MONone;
- const MDNode *AtomicStHint = I.getMetadata("aarch64.atomic.hint");
- if (AtomicStHint) {
- unsigned HintVal =
- cast<ConstantInt>(
- cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
- ->getZExtValue();
- AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
-
- if (static_cast<unsigned>(Hint) & 0b1)
- Flags |= MOAtomicHintBit0;
- if (static_cast<unsigned>(Hint) & 0b10)
- Flags |= MOAtomicHintBit1;
- }
-
- return Flags;
+ return MachineMemOperand::MONone;
}
bool AArch64TargetLowering::isLegalInterleavedAccessType(
@@ -33762,6 +33746,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
return true;
}
}
+
+ // The !mem.cache_hint metadata is not supported by GISel and will be dropped.
+ // TODO: Remove this and handle atomic store hints in GISel once supported.
+ if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
+ if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
+ return true;
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 8ab0c13e41edc..faddeab7868f4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -321,91 +321,93 @@ defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xexten
defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
-class atomic_hint_store<PatFrag Base, int Order, code Pred, code GIPred>
+class atomic_hint_store<PatFrag Base, int Order, code Pred>
: PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
let IsAtomicOrderingMonotonic = !eq(Order, 2);
let IsAtomicOrderingRelease = !eq(Order, 5);
let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
- let GISelPredicateCode = GIPred;
+
+ // TODO: Change once mem.cache_hint supported in GISel
+ let GISelPredicateCode = [{ return false; }];
}
-multiclass AtomicHintPatternsAddrMode<int Hint, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
(ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
(ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
(ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
(ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
(ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
(ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
(ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
(ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
(ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
(ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
(ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
(ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
}
-multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+multiclass AtomicHintPatterns<int Order, int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred, GIPred> GPR64:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred> GPR64:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
-
- defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
+
+ defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 25e0730149366..bd4674b40b9b8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -25,6 +25,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
@@ -2954,17 +2955,6 @@ bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
});
}
-AArch64AtomicStoreHint
-AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
- unsigned AtomicHint = 0;
- if (MMOFlags & MOAtomicHintBit0)
- AtomicHint += 0b1;
- if (MMOFlags & MOAtomicHintBit1)
- AtomicHint += 0b10;
-
- return static_cast<AArch64AtomicStoreHint>(AtomicHint);
-}
-
bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
switch (Opc) {
default:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 57bf875dd2a7f..9fd405396c424 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -31,10 +31,6 @@ static const MachineMemOperand::Flags MOSuppressPair =
MachineMemOperand::MOTargetFlag1;
static const MachineMemOperand::Flags MOStridedAccess =
MachineMemOperand::MOTargetFlag2;
-static const MachineMemOperand::Flags MOAtomicHintBit0 =
- MachineMemOperand::MOTargetFlag3;
-static const MachineMemOperand::Flags MOAtomicHintBit1 =
- MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
@@ -235,8 +231,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64AtomicStoreHint
- decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags);
+ static AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand &MMO);
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0f39bb84122da..5f9b31eeb05bd 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -527,11 +527,6 @@ class AArch64InstructionSelector : public InstructionSelector {
MachineOperand &Predicate,
MachineIRBuilder &MIRBuilder) const;
- bool isAtomicHintInst(const MachineInstr &MI,
- AArch64AtomicStoreHint Hint) const;
- bool isAtomicSTSHH_KEEP(const MachineInstr &MI) const;
- bool isAtomicSTSHH_STRM(const MachineInstr &MI) const;
-
/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
@@ -2524,107 +2519,6 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
I.eraseFromParent();
return true;
}
- case TargetOpcode::G_STORE: {
- GStore &St = cast<GStore>(I);
- auto MMO = St.getMMO();
- LLT PtrTy = MRI.getType(St.getPointerReg());
- AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
-
- // Only for handling atomic store with hint.
- // Can only handle AddressSpace 0, 64-bit pointers.
- if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64) ||
- Hint == AArch64AtomicStoreHint::HINT_NONE) {
- return false;
- }
-
- // AtomicExpandPass converts FP stores to integer stores of the equivalent
- // bitwidth. Widen the register here if the original type was f16.
- unsigned StoreSize = St.getMemSize().getValue();
- Register ValueReg = St.getValueReg();
- if (StoreSize == 2) {
- Register CastReg;
- if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
- auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
- {&AArch64::FPR32RegClass}, {});
- auto Ins = MIB.buildInstr(TargetOpcode::INSERT_SUBREG,
- {&AArch64::FPR32RegClass}, {Undef, ValueReg})
- .addImm(AArch64::hsub);
- constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
- constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
- ValueReg = Ins.getReg(0);
- }
- }
-
- static constexpr unsigned BaseOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_B, AArch64::ATOMIC_STORE_HINT_H,
- AArch64::ATOMIC_STORE_HINT_S, AArch64::ATOMIC_STORE_HINT_D};
- static constexpr unsigned RegWOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_BroW, AArch64::ATOMIC_STORE_HINT_HroW,
- AArch64::ATOMIC_STORE_HINT_SroW, AArch64::ATOMIC_STORE_HINT_DroW};
- static constexpr unsigned RegXOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_BroX, AArch64::ATOMIC_STORE_HINT_HroX,
- AArch64::ATOMIC_STORE_HINT_SroX, AArch64::ATOMIC_STORE_HINT_DroX};
- static constexpr unsigned UImmOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_Bui, AArch64::ATOMIC_STORE_HINT_Hui,
- AArch64::ATOMIC_STORE_HINT_Sui, AArch64::ATOMIC_STORE_HINT_Dui};
- static constexpr unsigned ImmOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_Bi, AArch64::ATOMIC_STORE_HINT_Hi,
- AArch64::ATOMIC_STORE_HINT_Si, AArch64::ATOMIC_STORE_HINT_Di};
-
- AtomicOrdering Ordering = MMO.getSuccessOrdering();
- MachineInstrBuilder StrPseudo;
- unsigned HintOpc = 0;
-
- // Other addressing modes can be used when the ordering is monotonic.
- // Try to match these first, before falling back on the basic operands.
- auto AddModeWRO = selectAddrModeWRO(St.getOperand(1), StoreSize);
- auto AddModeXRO = selectAddrModeXRO(St.getOperand(1), StoreSize);
- auto AddModeUImm = selectAddrModeIndexed(St.getOperand(1), StoreSize);
- auto AddModeImm = selectAddrModeUnscaled(St.getOperand(1), StoreSize);
-
- if (AddModeWRO && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = RegWOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeWRO)[0](StrPseudo);
- (*AddModeWRO)[1](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeWRO)[2](StrPseudo);
- } else if (AddModeXRO && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = RegXOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeXRO)[0](StrPseudo);
- (*AddModeXRO)[1](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeXRO)[2](StrPseudo);
- } else if (AddModeUImm && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = UImmOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeUImm)[0](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeUImm)[1](StrPseudo);
- } else if (AddModeImm && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = ImmOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeImm)[0](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeImm)[1](StrPseudo);
- } else {
- HintOpc = BaseOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
- .addReg(St.getPointerReg())
- .addReg(ValueReg);
- }
-
- // Add the ordering and hint operands, before erasing the store.
- unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 48 : 49;
- StrPseudo.addImm((int)Ordering);
- StrPseudo.addImm(HintImm);
-
- StrPseudo.cloneMemRefs(I);
- I.eraseFromParent();
- constrainSelectedInstRegOperands(*StrPseudo, TII, TRI, RBI);
- return true;
- }
default:
return false;
}
@@ -8254,22 +8148,6 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
.getZExtValue()));
}
-bool AArch64InstructionSelector::isAtomicHintInst(
- const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
- auto MMO = cast<GStore>(MI).getMMO();
- return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
-}
-
-bool AArch64InstructionSelector::isAtomicSTSHH_KEEP(
- const MachineInstr &MI) const {
- return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
-}
-
-bool AArch64InstructionSelector::isAtomicSTSHH_STRM(
- const MachineInstr &MI) const {
- return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_STRM);
-}
-
bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index 21aec7dae322f..d2c5b388df0b3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; STSHH: Keep, Relaxed
@@ -12,7 +12,7 @@ define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: strb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -22,7 +22,7 @@ define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: strh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -32,7 +32,7 @@ define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: str w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -42,7 +42,7 @@ define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: str x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -50,6 +50,18 @@ define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Keep, Release
;
+define void @test_atomic_store_keep_release_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
; CHECK: // %bb.0:
@@ -58,7 +70,7 @@ define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwi
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -70,7 +82,7 @@ define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -81,7 +93,7 @@ define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -92,7 +104,7 @@ define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwi
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -106,7 +118,7 @@ define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -116,7 +128,7 @@ define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -126,7 +138,7 @@ define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -136,7 +148,7 @@ define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -144,6 +156,18 @@ define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Stream, Relaxed
;
+define void @test_atomic_store_strm_relaxed_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
; CHECK: // %bb.0:
@@ -152,7 +176,7 @@ define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwi
; CHECK-NEXT: stshh strm
; CHECK-NEXT: strh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -164,7 +188,7 @@ define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: strh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -175,7 +199,7 @@ define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh strm
; CHECK-NEXT: str w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -186,7 +210,7 @@ define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwi
; CHECK-NEXT: stshh strm
; CHECK-NEXT: str x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -200,7 +224,7 @@ define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -210,7 +234,7 @@ define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -220,7 +244,7 @@ define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -230,7 +254,7 @@ define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -238,6 +262,18 @@ define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Stream, SequentiallyConsistent
;
+define void @test_atomic_store_stream_seqcst_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
; CHECK: // %bb.0:
@@ -246,7 +282,7 @@ define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounw
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -258,7 +294,7 @@ define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -269,7 +305,7 @@ define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -280,7 +316,7 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -293,10 +329,13 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !2
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4
ret void
}
-!0 = !{i32 0}
-!1 = !{i32 1}
-!2 = !{i32 2}
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.atomic_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.atomic_hint", i32 1}
+!4 = !{i32 1, !5}
+!5 = !{!"aarch64.atomic_hint", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index d80d9460c107c..a6dea2b9f511e 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -global-isel=1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; W register offset
@@ -15,7 +15,7 @@ define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind
; CHECK-NEXT: strb w2, [x0, w1, sxtw]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -26,7 +26,7 @@ define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwin
; CHECK-NEXT: strh w2, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 %offset
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -37,7 +37,7 @@ define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwin
; CHECK-NEXT: str w2, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 %offset
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -48,12 +48,25 @@ define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwin
; CHECK-NEXT: str x2, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 %offset
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
; FP
+define void @relaxed_store_hint_roW_mfloat(ptr %ptr, i32 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
; CHECK: // %bb.0:
@@ -63,7 +76,7 @@ define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) n
; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 %offset
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -76,7 +89,7 @@ define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounw
; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 %offset
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -88,7 +101,7 @@ define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nou
; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 %offset
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -100,7 +113,7 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n
; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 %offset
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -117,7 +130,7 @@ define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind
; CHECK-NEXT: strb w2, [x0, x1]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i64 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -128,7 +141,7 @@ define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwin
; CHECK-NEXT: strh w2, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i64 %offset
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -139,7 +152,7 @@ define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwin
; CHECK-NEXT: str w2, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i64 %offset
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -150,12 +163,25 @@ define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwin
; CHECK-NEXT: str x2, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i64 %offset
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
; FP
+define void @relaxed_store_hint_roX_mfloat(ptr %ptr, i64 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i64 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
; CHECK: // %bb.0:
@@ -165,7 +191,7 @@ define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) n
; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i64 %offset
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -178,7 +204,7 @@ define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounw
; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i64 %offset
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -190,7 +216,7 @@ define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nou
; CHECK-NEXT: str w8, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i64 %offset
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -202,7 +228,7 @@ define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) n
; CHECK-NEXT: str x8, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i64 %offset
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -219,7 +245,7 @@ define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: strb w1, [x0, #4095]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 4095
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -230,7 +256,7 @@ define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: strh w1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 2048
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -241,7 +267,7 @@ define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: str w1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 1024
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -252,12 +278,25 @@ define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: str x1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 512
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
; FP
+define void @relaxed_store_hint_uimm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, #2048]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 2048
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
; CHECK: // %bb.0:
@@ -267,7 +306,7 @@ define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-NEXT: strh w8, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 2048
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -280,7 +319,7 @@ define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: strh w8, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 2048
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -292,7 +331,7 @@ define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
; CHECK-NEXT: str w8, [x0, #8192]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 2048
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -304,7 +343,7 @@ define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
; CHECK-NEXT: str x8, [x0, #16384]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 2048
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -321,7 +360,7 @@ define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: sturb w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 -256
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -332,7 +371,7 @@ define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: sturh w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 -128
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -343,7 +382,7 @@ define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stur w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 -64
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -354,12 +393,25 @@ define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stur x1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 -32
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
; FP
+define void @relaxed_store_hint_imm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturb w8, [x0, #-128]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 -128
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
; CHECK: // %bb.0:
@@ -369,7 +421,7 @@ define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-NEXT: sturh w8, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 -128
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -382,7 +434,7 @@ define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: sturh w8, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 -128
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -395,7 +447,7 @@ define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
; CHECK-NEXT: str w9, [x8]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 -128
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -408,9 +460,11 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
; CHECK-NEXT: str x9, [x8]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 -128
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
-!0 = !{ i32 0 }
-!1 = !{ i32 1 }
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.atomic_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.atomic_hint", i32 1}
>From 26e4dcd4f97d944f388ac002849b1f98037cfa34 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 25 Aug 2026 14:53:01 +0000
Subject: [PATCH 7/8] - Reduce number of patterns by replacing Order with
Relaxed - Add floating-point patterns for relaxed stores - Renamed
aarch64.atomic_hint -> aarch64.mem_hint - Added warning for invalid types
passed to builtin for hint argument - Rename getAtomicStoreHintFromMD ->
toAArch64Hint - Cleaned up includes
---
.../clang/Basic/DiagnosticSemaKinds.td | 2 +
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 10 +-
clang/lib/Sema/SemaARM.cpp | 13 +-
clang/test/CodeGen/arm_acle.c | 4 +-
clang/test/CodeGen/builtins-arm64.c | 4 +-
clang/test/Sema/builtins-arm64.c | 4 +-
.../include/llvm/Support/AArch64AtomicHints.h | 36 -----
.../include/llvm/Support/AArch64MemoryHints.h | 35 +++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 66 +++++---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 36 ++---
.../lib/Target/AArch64/AArch64InstrAtomics.td | 148 ++++++++++--------
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 1 -
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 4 +-
.../GISel/AArch64InstructionSelector.cpp | 1 -
.../Atomics/aarch64-atomic-store-hint.ll | 12 +-
.../Atomics/aarch64-relaxed-store-hint.ll | 28 ++--
16 files changed, 221 insertions(+), 183 deletions(-)
delete mode 100644 llvm/include/llvm/Support/AArch64AtomicHints.h
create mode 100644 llvm/include/llvm/Support/AArch64MemoryHints.h
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 5c7d696642583..145625db3900a 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9656,6 +9656,8 @@ def err_atomic_builtin_must_be_pointer_intptr : Error<
def err_atomic_hint_builtin_must_be_pointer : Error<
"address argument to atomic hint builtin must be a pointer to a scalar "
"integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+def err_atomic_hint_has_invalid_hint_type : Error<
+ "invalid hint type argument to atomic hint operation (%0)">;
def err_atomic_builtin_cannot_be_const : Error<
"address argument to atomic builtin cannot be const-qualified (%0 invalid)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 8ab9ecb65b4c4..ae9ad731008f5 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -20,7 +20,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include <numeric>
@@ -2105,14 +2105,14 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned HintArg = Result.Val.getInt().getExtValue();
// Attach the hint if valid
- if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
+ if (toAArch64Hint(HintArg) != AArch64MemoryHint::HINT_NONE) {
LLVMContext &Ctx = CGM.getLLVMContext();
- MDNode *AtomicHint = MDNode::get(
- Ctx, {MDString::get(Ctx, "aarch64.atomic_hint"),
+ MDNode *MemHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
MDNode *HintNode = MDNode::get(
CGM.getLLVMContext(),
- {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), AtomicHint});
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
}
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index b3c139bf1c8bc..e68f4868e7698 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -17,7 +17,7 @@
#include "clang/Sema/Initialization.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
namespace clang {
@@ -343,7 +343,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
- if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType() &&
+ if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
!PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_op_needs_atomic_int_or_fp)
@@ -391,20 +391,19 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_hint_has_invalid_memory_order)
<< *MemOrdAP << MemOrdArg->getSourceRange();
- // Arg 3 is the hint type. Only values represented by AArch64AtomicStoreHint
+ // Arg 3 is the hint type. Only values represented by AArch64MemoryHint
// are valid.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP) {
- Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ if (!HintAP || !HintArg->getType()->isIntegralType(Context)) {
+ Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
return false;
}
unsigned Hint = HintAP->getZExtValue();
- if (llvm::getAtomicStoreHintFromMD(Hint) ==
- llvm::AArch64AtomicStoreHint::HINT_NONE)
+ if (llvm::toAArch64Hint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 041530a5d2269..e8e9d4ee72534 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1936,8 +1936,8 @@ void test_atomic_store_hint_double(double *ptr, double data) {
}
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
-// AArch64-NEXT: [[HINT2]] = !{!"aarch64.atomic_hint", i32 0}
+// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
-// AArch64-NEXT: [[HINT4]] = !{!"aarch64.atomic_hint", i32 1}
+// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index ed2bb6e65b525..7464e66400388 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -233,6 +233,6 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
-// CHECK: ![[M2]] = !{!"aarch64.atomic_hint", i32 0}
+// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0}
// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
-// CHECK: ![[M4]] = !{!"aarch64.atomic_hint", i32 1}
+// CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 5372aff24b739..c665507220a67 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -65,6 +65,8 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-warning {{unrecognised hint type argument to atomic hint operation ('int')}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{invalid hint type argument to atomic hint operation ('char *')}}
}
diff --git a/llvm/include/llvm/Support/AArch64AtomicHints.h b/llvm/include/llvm/Support/AArch64AtomicHints.h
deleted file mode 100644
index 8118f3e2df3ad..0000000000000
--- a/llvm/include/llvm/Support/AArch64AtomicHints.h
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- AArch64AtomicHints.h - AArch64 Atomic Hint Attributes ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_AARCH64ATOMICHINTS_H
-#define LLVM_SUPPORT_AARCH64ATOMICHINTS_H
-
-namespace llvm {
-enum class AArch64AtomicStoreHint {
- HINT_NONE = 0,
- HINT_STSHH_KEEP = 1,
- HINT_STSHH_STRM = 2,
-};
-
-template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
- return (Int)AArch64AtomicStoreHint::HINT_STSHH_KEEP <= I &&
- I <= (Int)AArch64AtomicStoreHint::HINT_STSHH_STRM;
-}
-
-template <typename Int>
-inline AArch64AtomicStoreHint getAtomicStoreHintFromMD(Int I) {
- switch (I) {
- case 0:
- return AArch64AtomicStoreHint::HINT_STSHH_KEEP;
- case 1:
- return AArch64AtomicStoreHint::HINT_STSHH_STRM;
- default:
- return AArch64AtomicStoreHint::HINT_NONE;
- }
-}
-} // namespace llvm
-#endif // LLVM_SUPPORT_AARCH64ATOMICHINTS_H
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
new file mode 100644
index 0000000000000..9efc68faf0718
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -0,0 +1,35 @@
+//===-- AArch64MemoryHints.h - AArch64 Memory Hint Attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+#define LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+
+namespace llvm {
+enum class AArch64MemoryHint {
+ HINT_NONE = 0,
+ HINT_STSHH_KEEP = 1,
+ HINT_STSHH_STRM = 2,
+};
+
+template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+ return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I &&
+ I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM;
+}
+
+template <typename Int> inline AArch64MemoryHint toAArch64Hint(Int I) {
+ switch (I) {
+ case 0:
+ return AArch64MemoryHint::HINT_STSHH_KEEP;
+ case 1:
+ return AArch64MemoryHint::HINT_STSHH_STRM;
+ default:
+ return AArch64MemoryHint::HINT_NONE;
+ }
+}
+} // namespace llvm
+#endif // LLVM_SUPPORT_AARCH64MEMORYHINTS_H
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c7277637f3cb5..60ff2fae05d64 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3259,8 +3259,8 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(2).getImm();
- bool Relaxed = Order == 2;
+ unsigned Relaxed = MI->getOperand(2).getImm();
+ assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
switch (MI->getOpcode()) {
case AArch64::ATOMIC_STORE_HINT_B:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
@@ -3268,10 +3268,10 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_H:
StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
break;
- case AArch64::ATOMIC_STORE_HINT_S:
+ case AArch64::ATOMIC_STORE_HINT_W:
StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
break;
- case AArch64::ATOMIC_STORE_HINT_D:
+ case AArch64::ATOMIC_STORE_HINT_X:
StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
break;
default:
@@ -3294,8 +3294,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(5).getImm();
- assert(Order == 2 &&
+ assert(MI->getOperand(5).getImm() == 1 &&
"Atomic store addressing mode only supports relaxed stores");
switch (MI->getOpcode()) {
@@ -3305,24 +3304,36 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
case AArch64::ATOMIC_STORE_HINT_HroW:
StOpc = AArch64::STRHHroW;
break;
- case AArch64::ATOMIC_STORE_HINT_SroW:
+ case AArch64::ATOMIC_STORE_HINT_WroW:
StOpc = AArch64::STRWroW;
break;
- case AArch64::ATOMIC_STORE_HINT_DroW:
+ case AArch64::ATOMIC_STORE_HINT_XroW:
StOpc = AArch64::STRXroW;
break;
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ StOpc = AArch64::STRSroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ StOpc = AArch64::STRDroW;
+ break;
case AArch64::ATOMIC_STORE_HINT_BroX:
StOpc = AArch64::STRBBroX;
break;
case AArch64::ATOMIC_STORE_HINT_HroX:
StOpc = AArch64::STRHHroX;
break;
- case AArch64::ATOMIC_STORE_HINT_SroX:
+ case AArch64::ATOMIC_STORE_HINT_WroX:
StOpc = AArch64::STRWroX;
break;
- case AArch64::ATOMIC_STORE_HINT_DroX:
+ case AArch64::ATOMIC_STORE_HINT_XroX:
StOpc = AArch64::STRXroX;
break;
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ StOpc = AArch64::STRSroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ StOpc = AArch64::STRDroX;
+ break;
default:
llvm_unreachable("Unexpected atomic hint opcode.");
}
@@ -3344,8 +3355,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(3).getImm();
- assert(Order == 2 &&
+ assert(MI->getOperand(3).getImm() == 1 &&
"Atomic store addressing mode only supports relaxed stores");
switch (MI->getOpcode()) {
@@ -3355,24 +3365,36 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
case AArch64::ATOMIC_STORE_HINT_Hui:
StOpc = AArch64::STRHHui;
break;
- case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
StOpc = AArch64::STRWui;
break;
- case AArch64::ATOMIC_STORE_HINT_Dui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
StOpc = AArch64::STRXui;
break;
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ StOpc = AArch64::STRSui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ StOpc = AArch64::STRDui;
+ break;
case AArch64::ATOMIC_STORE_HINT_Bi:
StOpc = AArch64::STURBBi;
break;
case AArch64::ATOMIC_STORE_HINT_Hi:
StOpc = AArch64::STURHHi;
break;
- case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
StOpc = AArch64::STURWi;
break;
- case AArch64::ATOMIC_STORE_HINT_Di:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
StOpc = AArch64::STURXi;
break;
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ StOpc = AArch64::STURSi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ StOpc = AArch64::STURDi;
+ break;
default:
llvm_unreachable("Unexpected atomic hint opcode.");
}
@@ -4099,26 +4121,34 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
return;
case AArch64::ATOMIC_STORE_HINT_B:
case AArch64::ATOMIC_STORE_HINT_H:
- case AArch64::ATOMIC_STORE_HINT_S:
- case AArch64::ATOMIC_STORE_HINT_D:
+ case AArch64::ATOMIC_STORE_HINT_W:
+ case AArch64::ATOMIC_STORE_HINT_X:
emitAtomicHintPseudoExpansion(MI);
return;
case AArch64::ATOMIC_STORE_HINT_BroW:
case AArch64::ATOMIC_STORE_HINT_HroW:
+ case AArch64::ATOMIC_STORE_HINT_WroW:
+ case AArch64::ATOMIC_STORE_HINT_XroW:
case AArch64::ATOMIC_STORE_HINT_SroW:
case AArch64::ATOMIC_STORE_HINT_DroW:
case AArch64::ATOMIC_STORE_HINT_BroX:
case AArch64::ATOMIC_STORE_HINT_HroX:
+ case AArch64::ATOMIC_STORE_HINT_WroX:
+ case AArch64::ATOMIC_STORE_HINT_XroX:
case AArch64::ATOMIC_STORE_HINT_SroX:
case AArch64::ATOMIC_STORE_HINT_DroX:
emitAtomicHintPseudoExpansionRO(MI);
return;
case AArch64::ATOMIC_STORE_HINT_Bui:
case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
case AArch64::ATOMIC_STORE_HINT_Sui:
case AArch64::ATOMIC_STORE_HINT_Dui:
case AArch64::ATOMIC_STORE_HINT_Bi:
case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
case AArch64::ATOMIC_STORE_HINT_Si:
case AArch64::ATOMIC_STORE_HINT_Di:
emitAtomicHintPseudoExpansionImm(MI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index bd00fdceb8bab..5cd7d8eb1b7eb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -24,7 +24,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
@@ -518,7 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
- AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand *MMO) const;
+ AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
bool isAtomicSTSHH_KEEP(SDNode *N) const;
bool isAtomicSTSHH_STRM(SDNode *N) const;
@@ -4616,36 +4616,32 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
-AArch64AtomicStoreHint
-AArch64DAGToDAGISel::decodeAtomicHintFlags(MachineMemOperand *MMO) const {
- int AtomicHint = -1;
+AArch64MemoryHint
+AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
+ int MemoryHint = -1;
const MDNode *MemCacheHint = MMO->getMemCacheHint();
if (!MemCacheHint)
- return AArch64AtomicStoreHint::HINT_NONE;
+ return AArch64MemoryHint::HINT_NONE;
for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
- const auto *Key = cast<MDString>(MemCacheHint->getOperand(I));
- StringRef KeyStr = Key->getString();
- const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
-
- if (KeyStr != "aarch64.atomic_hint")
- continue;
-
- AtomicHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
- ->getZExtValue();
+ if (MemCacheHint->getOperand(I).equalsStr("aarch64.mem_hint")) {
+ const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
+ MemoryHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
+ ->getZExtValue();
+ }
}
- return getAtomicStoreHintFromMD(AtomicHint);
+ return toAArch64Hint(MemoryHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
- return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64AtomicStoreHint::HINT_STSHH_KEEP;
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_KEEP;
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
- return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64AtomicStoreHint::HINT_STSHH_STRM;
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_STRM;
}
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index faddeab7868f4..e0e65c400a8b0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -289,23 +289,23 @@ def : Pat<(relaxed_store<atomic_store_64>
let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
: Pseudo<(outs), (ins GPR64sp:$Rn, GPR32:$Rm, regtype:$data, WExt:$extend,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
: Pseudo<(outs), (ins GPR64sp:$Rn, GPR64:$Rm, regtype:$data, XExt:$extend,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
: Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, UImmOp:$offset,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoImm<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, simm9:$offset,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
}
multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> {
@@ -318,96 +318,118 @@ multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand X
defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>;
defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
-defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
-defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_W : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_X : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<FPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<FPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
-class atomic_hint_store<PatFrag Base, int Order, code Pred>
+class atomic_hint_store<PatFrag Base, int Relaxed, code Pred>
: PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
- let IsAtomicOrderingMonotonic = !eq(Order, 2);
- let IsAtomicOrderingRelease = !eq(Order, 5);
- let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
+ let IsAtomicOrderingReleaseOrStronger = !eq(Relaxed, 0);
// TODO: Change once mem.cache_hint supported in GISel
let GISelPredicateCode = [{ return false; }];
}
multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
- (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
- (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
- (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_WroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
- (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_XroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, FPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, FPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
- (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
- (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
- (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_WroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
- (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_XroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, FPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, FPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
- (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
- (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
- (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_Wui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
- (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_Xui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, FPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, FPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_Wi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
-}
-
-multiclass AtomicHintPatterns<int Order, int Hint, code Pred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred> GPR64:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Xi GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si GPR64sp:$addr, FPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di GPR64sp:$addr, FPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+}
+
+multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_W GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_X GPR64sp:$addr, GPR64:$data, (i32 Relaxed), (i32 Hint))>;
}
let AddedComplexity = 15 in {
defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index bd4674b40b9b8..fe0c59622e17b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -25,7 +25,6 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
-#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 9fd405396c424..ab27c57cbd192 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -16,7 +16,7 @@
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/TypeSize.h"
#include <optional>
@@ -231,7 +231,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand &MMO);
+ static AArch64MemoryHint decodeMemoryFlags(MachineMemOperand &MMO);
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 5f9b31eeb05bd..0b018dc7b22a8 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -45,7 +45,6 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Type.h"
#include "llvm/Pass.h"
-#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index d2c5b388df0b3..bfa9fc49cf752 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -195,9 +195,8 @@ define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w8, [x0]
+; CHECK-NEXT: str s0, [x0]
; CHECK-NEXT: ret
store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
@@ -206,9 +205,8 @@ define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind
define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x8, [x0]
+; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret
store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
@@ -334,8 +332,8 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
}
!0 = !{i32 1, !1}
-!1 = !{!"aarch64.atomic_hint", i32 0}
+!1 = !{!"aarch64.mem_hint", i32 0}
!2 = !{i32 1, !3}
-!3 = !{!"aarch64.atomic_hint", i32 1}
+!3 = !{!"aarch64.mem_hint", i32 1}
!4 = !{i32 1, !5}
-!5 = !{!"aarch64.atomic_hint", i32 2}
+!5 = !{!"aarch64.mem_hint", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index a6dea2b9f511e..5f5bc43039778 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -96,9 +96,8 @@ define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounw
define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
+; CHECK-NEXT: str s0, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 %offset
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -108,9 +107,8 @@ define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nou
define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
+; CHECK-NEXT: str d0, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 %offset
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -211,9 +209,8 @@ define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounw
define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str w8, [x0, x1, lsl #2]
+; CHECK-NEXT: str s0, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i64 %offset
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -223,9 +220,8 @@ define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nou
define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str x8, [x0, x1, lsl #3]
+; CHECK-NEXT: str d0, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i64 %offset
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -326,9 +322,8 @@ define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str w8, [x0, #8192]
+; CHECK-NEXT: str s0, [x0, #8192]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 2048
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -338,9 +333,8 @@ define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str x8, [x0, #16384]
+; CHECK-NEXT: str d0, [x0, #16384]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 2048
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -441,10 +435,9 @@ define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w9, s0
; CHECK-NEXT: sub x8, x0, #512
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w9, [x8]
+; CHECK-NEXT: str s0, [x8]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 -128
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -454,10 +447,9 @@ define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x9, d0
; CHECK-NEXT: sub x8, x0, #1024
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x9, [x8]
+; CHECK-NEXT: str d0, [x8]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 -128
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -465,6 +457,6 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
}
!0 = !{i32 1, !1}
-!1 = !{!"aarch64.atomic_hint", i32 0}
+!1 = !{!"aarch64.mem_hint", i32 0}
!2 = !{i32 1, !3}
-!3 = !{!"aarch64.atomic_hint", i32 1}
+!3 = !{!"aarch64.mem_hint", i32 1}
>From baa6fc55f9f2123bf5aa524207e1a3b562e6dfdb Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 2 Sep 2026 10:57:36 +0000
Subject: [PATCH 8/8] - Remove unnecessary headers & decodeMemoryFlags
declaration
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 2 +-
clang/lib/Sema/SemaARM.cpp | 4 ++--
llvm/include/llvm/Support/AArch64MemoryHints.h | 4 ++--
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 3 +--
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 --
5 files changed, 6 insertions(+), 9 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index ae9ad731008f5..37d43cb4c98ca 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2105,7 +2105,7 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned HintArg = Result.Val.getInt().getExtValue();
// Attach the hint if valid
- if (toAArch64Hint(HintArg) != AArch64MemoryHint::HINT_NONE) {
+ if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::HINT_NONE) {
LLVMContext &Ctx = CGM.getLLVMContext();
MDNode *MemHint = MDNode::get(
Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index e68f4868e7698..1159b527c9a9e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -396,14 +396,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP || !HintArg->getType()->isIntegralType(Context)) {
+ if (!HintAP) {
Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
return false;
}
unsigned Hint = HintAP->getZExtValue();
- if (llvm::toAArch64Hint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
+ if (llvm::toAArch64MemoryHint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
index 9efc68faf0718..44ac64e481e80 100644
--- a/llvm/include/llvm/Support/AArch64MemoryHints.h
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -16,12 +16,12 @@ enum class AArch64MemoryHint {
HINT_STSHH_STRM = 2,
};
-template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I &&
I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM;
}
-template <typename Int> inline AArch64MemoryHint toAArch64Hint(Int I) {
+template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
switch (I) {
case 0:
return AArch64MemoryHint::HINT_STSHH_KEEP;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 5cd7d8eb1b7eb..99911c94c7f1f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -12,7 +12,6 @@
#include "AArch64.h"
#include "AArch64ExpandImm.h"
-#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
@@ -4631,7 +4630,7 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
}
}
- return toAArch64Hint(MemoryHint);
+ return toAArch64MemoryHint(MemoryHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index ab27c57cbd192..558a401e7bd44 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -231,8 +231,6 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64MemoryHint decodeMemoryFlags(MachineMemOperand &MMO);
-
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
static bool hasUnscaledLdStOffset(MachineInstr &MI) {
More information about the cfe-commits
mailing list