[clang] [llvm] [AArch64] Implement the atomic store with hint intrinsic (PR #198316)
Kerry McLaughlin via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 02:54:49 PDT 2026
https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/198316
>From 2efbb40aeeb3ba31724650eeaf282982a5c55bea Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 6 May 2026 13:08:02 +0000
Subject: [PATCH 01/13] [AArch64] Implement the atomic store with hint
intrinsic
Adds the following ACLE intrinsic as described in [1]:
void __arm_atomic_store_with_hint(type *ptr, type data,
int memory_order, int hint);
A regular atomic store instruction is emitted in Clang for this builtin
with additional metadata (`!aarch64.atomic.hint`), which ensures the
instruction is recognised as atomic by passes in LLVM.
When an atomic store has this metadata, this lowers to the ATOMIC_STORE_HINT
pseudo which is later expanded by AArch64ExpandPseudoInsts into an STSHH
instruction plus an atomic store.
The hint value is represented using MOTargetFlag3 & MOTargetFlag4 flags,
which will need to be extended when new hints are added in future.
[1] https://github.com/ARM-software/acle/pull/432
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 +
.../clang/Basic/DiagnosticSemaKinds.td | 6 +
clang/include/clang/Sema/SemaARM.h | 1 +
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 54 ++++
clang/lib/Headers/arm_acle.h | 6 +
clang/lib/Sema/SemaARM.cpp | 92 ++++++
clang/test/CodeGen/arm_acle.c | 78 +++++
clang/test/CodeGen/builtins-arm64.c | 13 +
clang/test/Sema/builtins-arm64.c | 17 ++
.../include/llvm/Support/AArch64AtomicHints.h | 36 +++
llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 +
.../AArch64/AArch64ExpandPseudoInsts.cpp | 56 ++++
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 20 ++
.../Target/AArch64/AArch64ISelLowering.cpp | 21 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 109 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 13 +
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 9 +
.../GISel/AArch64InstructionSelector.cpp | 61 ++++
.../Atomics/aarch64-atomic-store-hint.ll | 288 ++++++++++++++++++
.../Atomics/aarch64-atomic-store-hint.mir | 30 ++
20 files changed, 916 insertions(+), 1 deletion(-)
create mode 100644 llvm/include/llvm/Support/AArch64AtomicHints.h
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 4c49c874133fb..30aa3d526cbbb 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -171,6 +171,10 @@ let Attributes = [NoThrow], Features = "ls64" in {
def st64bv0 : AArch64TargetBuiltin<"uint64_t (void *, uint64_t const *)">;
}
+let Attributes = [NoThrow, CustomTypeChecking] in {
+ def atomic_store_with_hint : AArch64Builtin<"void(...)">;
+}
+
// Armv9.3-A Guarded Control Stack
let Attributes = [NoThrow], Features = "gcs" in {
def gcspopm : AArch64TargetBuiltin<"uint64_t (uint64_t)">;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 36a18473f4d4c..11a2ccfd1e0b1 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9683,6 +9683,12 @@ def err_atomic_op_needs_atomic_int : Error<
def err_atomic_op_needs_atomic_fp
: Error<"address argument to atomic operation must be a pointer to "
"%select{|atomic }0floating point type (%1 invalid)">;
+def err_atomic_op_hint_data_size : Error<
+ "address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
+def err_atomic_hint_has_invalid_memory_order : Error<
+ "invalid memory order argument to atomic hint operation (%0 invalid)">;
+def err_atomic_hint_has_invalid_hint_type : Error<
+ "invalid hint type argument to atomic hint operation (%0 invalid)">;
def warn_atomic_op_has_invalid_memory_order : Warning<
"%select{|success |failure }0memory order argument to atomic operation is invalid">,
InGroup<DiagGroup<"atomic-memory-ordering">>;
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index af8e0e9047171..b0a01c40ffece 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -70,6 +70,7 @@ class SemaARM : public SemaBase {
bool BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall, int ArgNum,
unsigned ExpectedFieldNum, bool AllowName);
bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
+ bool BuiltinARMAtomicStoreHintCall(unsigned BuiltinID, CallExpr *TheCall);
bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 621e5d18ebfee..efe742e3bc091 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -20,6 +20,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include <numeric>
@@ -2065,6 +2066,56 @@ static Value *EmitRangePrefetchBuiltin(CodeGenFunction &CGF, unsigned BuiltinID,
Ops);
}
+static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
+ unsigned BuiltinID,
+ const CallExpr *E) {
+ CodeGen::CGBuilderTy &Builder = CGF.Builder;
+ CodeGen::CodeGenModule &CGM = CGF.CGM;
+ Expr::EvalResult Result;
+ if (!E->getArg(2)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer policy argument to atomic store with hint.");
+
+ StoreInst *Store =
+ Builder.CreateStore(CGF.EmitScalarExpr(E->getArg(1)), // Value
+ CGF.EmitPointerWithAlignment(E->getArg(0))); // Ptr;
+
+ AtomicOrdering Ordering;
+ unsigned OrderingArg = Result.Val.getInt().getExtValue();
+ assert(isValidAtomicOrderingCABI(OrderingArg) && "Invalid atomic ordering");
+
+ switch (static_cast<AtomicOrderingCABI>(OrderingArg)) {
+ default:
+ llvm_unreachable("Unsupported atomic ordering found.");
+ case AtomicOrderingCABI::relaxed:
+ Ordering = AtomicOrdering::Monotonic;
+ break;
+ case AtomicOrderingCABI::release:
+ Ordering = AtomicOrdering::Release;
+ break;
+ case AtomicOrderingCABI::seq_cst:
+ Ordering = AtomicOrdering::SequentiallyConsistent;
+ break;
+ }
+ Store->setAtomic(Ordering);
+
+ if (!E->getArg(3)->EvaluateAsInt(Result, CGM.getContext()))
+ llvm_unreachable(
+ "Expected integer hint argument to atomic store with hint.");
+ unsigned HintArg = Result.Val.getInt().getExtValue();
+ assert((getAtomicStoreHintFromMD(HintArg) !=
+ AArch64AtomicStoreHint::HINT_NONE) &&
+ "Invalid hint type");
+
+ MDNode *HintMDVal =
+ MDNode::get(CGM.getLLVMContext(),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
+ Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
+ HintMDVal);
+
+ return Store;
+}
+
/// Return true if BuiltinID is an overloaded Neon intrinsic with an extra
/// argument that specifies the vector type. The additional argument is meant
/// for Sema checking (see `CheckNeonBuiltinFunctionCall`) and this function
@@ -4840,6 +4891,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__builtin_arm_range_prefetch_x)
return EmitRangePrefetchBuiltin(*this, BuiltinID, E);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return EmitAtomicStoreWithHintBuiltin(*this, BuiltinID, E);
+
// Memory Tagging Extensions (MTE) Intrinsics
Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
switch (BuiltinID) {
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 73b1e841b6719..8d4fbdc2e8c32 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -742,6 +742,12 @@ __arm_st64bv0(void *__addr, data512_t __value) {
}
#endif
+/* Atomic store with hints */
+#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
+#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
+ __builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
+#endif
+
/* 11.1 Special register intrinsics */
#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f38a35ec08fb0..7a03f25a7e5c9 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -17,6 +17,7 @@
#include "clang/Sema/Initialization.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
+#include "llvm/Support/AArch64AtomicHints.h"
namespace clang {
@@ -322,6 +323,94 @@ bool SemaARM::BuiltinARMSpecialReg(unsigned BuiltinID, CallExpr *TheCall,
return false;
}
+bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
+ CallExpr *TheCall) {
+ if (SemaRef.checkArgCount(TheCall, 4))
+ return true;
+
+ // Arg 0 should be the pointer type. The pointee type must be a
+ // scalar integral or floating-point type of 8, 16, 32 or 64 bits.
+ ASTContext &Context = getASTContext();
+ Expr *PtrArg = TheCall->getArg(0);
+ auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(PtrArg);
+ if (PtrArgRes.isInvalid())
+ return true;
+ auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
+ if (!PtrTy)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_builtin_must_be_pointer)
+ << PtrArg->getType() << 0 << PtrArg->getSourceRange();
+ QualType PtrQT = PtrTy->getPointeeType();
+
+ // TODO: Allow MFloat8 types when supported by atomic store
+ if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_atomic_int_or_fp)
+ << 0 << PtrQT << PtrArg->getSourceRange();
+
+ unsigned TySize =
+ Context.getTypeSize(Context.getCanonicalType(PtrQT).getUnqualifiedType());
+ if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
+ return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
+ << PtrArg->getSourceRange();
+
+ // Arg 1 is the data to be stored. The type must match the pointee
+ // type found above.
+ auto DataArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
+ if (DataArgRes.isInvalid())
+ return true;
+ QualType DataQT = DataArgRes.get()->getType();
+
+ if (PtrQT != DataQT)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_typecheck_call_different_arg_types)
+ << PtrQT << DataQT;
+
+ // Arg 2 is the memory order, which must be relaxed, release or seq_cst
+ auto MemOrdArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)).get();
+ std::optional<llvm::APSInt> MemOrdAP =
+ MemOrdArg->getIntegerConstantExpr(Context);
+ if (!MemOrdAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << MemOrdArg->getType() << MemOrdArg->getSourceRange();
+
+ unsigned Ordering = MemOrdAP->getZExtValue();
+ if (!llvm::isValidAtomicOrderingCABI(Ordering))
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+
+ auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
+ if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::release &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+
+ // Arg 3 is the hint type. Only values represented by AArch64AtomicStoreHint
+ // are valid.
+ auto HintArg =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
+ std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
+ if (!HintAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << HintArg->getType() << HintArg->getSourceRange();
+
+ unsigned Hint = HintAP->getZExtValue();
+ if (llvm::getAtomicStoreHintFromMD(Hint) ==
+ llvm::AArch64AtomicStoreHint::HINT_NONE)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << *HintAP << HintArg->getSourceRange();
+
+ return false;
+}
+
/// getNeonEltType - Return the QualType corresponding to the elements of
/// the vector type specified by the NeonTypeFlags. This is used to check
/// the pointer arguments for Neon load/store intrinsics.
@@ -1170,6 +1259,9 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
BuiltinID == AArch64::BI__builtin_arm_wsrp)
return BuiltinARMSpecialReg(BuiltinID, TheCall, 0, 5, true);
+ if (BuiltinID == AArch64::BI__builtin_arm_atomic_store_with_hint)
+ return BuiltinARMAtomicStoreHintCall(BuiltinID, TheCall);
+
// Only check the valid encoding range. Any constant in this range would be
// converted to a register of the form S2_2_C3_C4_5. Let the hardware throw
// an exception for incorrect registers. This matches MSVC behavior.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index e1bbc3d745bcd..29e35ed56fa69 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1761,3 +1761,81 @@ int test_rndrrs(uint64_t *__addr) {
return __rndrrs(__addr);
}
#endif
+
+#if defined(__ARM_64BIT_STATE)
+
+// AArch64-LABEL: @test_atomic_store_hint_char(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_char(char *ptr, char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_bfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_short(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_short(short *ptr, short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_u32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_float(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_float(float *ptr, float data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_s64(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_double(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_double(double *ptr, double data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+}
+
+// AArch64: [[META3]] = !{i32 0}
+// AArch64-NEXT: [[META4]] = !{i32 1}
+#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 1800ff77a57a5..74553659fbfed 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -216,4 +216,17 @@ void trap() {
__builtin_arm_trap(42);
}
+void atomic_store_with_hint(int64_t *a, int64_t b) {
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !aarch64.atomic.hint ![[M1:[0-9]]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, 0);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !aarch64.atomic.hint ![[M1]]
+
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+}
+
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
+// CHECK: ![[M1]] = !{i32 0}
+// CHECK: ![[M2]] = !{i32 1}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 41cffd7ebb1a0..fb4718a1bd1f4 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -51,3 +51,20 @@ void test_trap(short s, unsigned short us) {
__builtin_arm_trap(s); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
__builtin_arm_trap(us); // expected-error {{argument to '__builtin_arm_trap' must be a constant integer}}
}
+
+void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
+ char c_data, __int128 inv_data, float f_data,
+ int inv_int) {
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
+
+ __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
+ __builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-error {{invalid hint type argument to atomic hint operation (3 invalid)}}
+}
diff --git a/llvm/include/llvm/Support/AArch64AtomicHints.h b/llvm/include/llvm/Support/AArch64AtomicHints.h
new file mode 100644
index 0000000000000..8118f3e2df3ad
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64AtomicHints.h
@@ -0,0 +1,36 @@
+//===-- AArch64AtomicHints.h - AArch64 Atomic Hint Attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64ATOMICHINTS_H
+#define LLVM_SUPPORT_AARCH64ATOMICHINTS_H
+
+namespace llvm {
+enum class AArch64AtomicStoreHint {
+ HINT_NONE = 0,
+ HINT_STSHH_KEEP = 1,
+ HINT_STSHH_STRM = 2,
+};
+
+template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+ return (Int)AArch64AtomicStoreHint::HINT_STSHH_KEEP <= I &&
+ I <= (Int)AArch64AtomicStoreHint::HINT_STSHH_STRM;
+}
+
+template <typename Int>
+inline AArch64AtomicStoreHint getAtomicStoreHintFromMD(Int I) {
+ switch (I) {
+ case 0:
+ return AArch64AtomicStoreHint::HINT_STSHH_KEEP;
+ case 1:
+ return AArch64AtomicStoreHint::HINT_STSHH_STRM;
+ default:
+ return AArch64AtomicStoreHint::HINT_NONE;
+ }
+}
+} // namespace llvm
+#endif // LLVM_SUPPORT_AARCH64ATOMICHINTS_H
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 50e09e40f6c75..d93d01be94709 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -257,6 +257,8 @@ static void copyMetadataForAtomic(Instruction &Dest,
Dest.setMetadata(ID, N);
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
+ else if (ID == Ctx.getMDKindID("aarch64.atomic.hint"))
+ Dest.setMetadata(ID, N);
// Losing atomic.ignore.denormal.mode, but it doesn't matter for current
// uses.
@@ -759,6 +761,7 @@ StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {
Value *Addr = SI->getPointerOperand();
StoreInst *NewSI = Builder.CreateStore(NewVal, Addr, SI->getProperties());
+ copyMetadataForAtomic(*NewSI, *SI);
LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");
SI->eraseFromParent();
return NewSI;
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 742721a2df16d..f07bc127318d5 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -68,6 +68,9 @@ class AArch64ExpandPseudoImpl {
MachineBasicBlock::iterator MBBI);
bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
+ bool expandAtomicStoreHintPseudo(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ unsigned Size);
bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
unsigned ExtendImm, unsigned ZeroReg,
@@ -1309,6 +1312,51 @@ bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
return true;
}
+bool AArch64ExpandPseudoImpl::expandAtomicStoreHintPseudo(
+ MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Size) {
+ MachineInstr &MI = *MBBI;
+ DebugLoc DL = MI.getDebugLoc();
+
+ unsigned StOpc;
+ unsigned Order = MI.getOperand(2).getImm();
+ bool Relaxed = Order == 0;
+
+ switch (Size) {
+ case 8:
+ StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
+ break;
+ case 16:
+ StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
+ break;
+ case 32:
+ StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
+ break;
+ case 64:
+ StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint size.");
+ }
+
+ auto *Hint = BuildMI(MBB, MBBI, DL, TII->get(AArch64::STSHH))
+ .addImm(MI.getOperand(3).getImm())
+ .getInstr();
+
+ auto Store = BuildMI(MBB, MBBI, DL, TII->get(StOpc))
+ .add(MI.getOperand(1))
+ .addReg(MI.getOperand(0).getReg())
+ .setMemRefs(MI.memoperands())
+ .setMIFlags(MI.getFlags());
+
+ if (Relaxed)
+ Store.addImm(0);
+
+ transferImpOps(MI, Store, Store);
+ finalizeBundle(MBB, Hint->getIterator(), MBBI->getIterator());
+ MI.eraseFromParent();
+ return true;
+}
+
/// If MBBI references a pseudo instruction that should be expanded here,
/// do the expansion and return true. Otherwise return false.
bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
@@ -2011,6 +2059,14 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case AArch64::NAND_ZZZ:
case AArch64::NOR_ZZZ:
return expandSVEBitwisePseudo(MI, MBB, MBBI);
+ case AArch64::ATOMIC_STORE_HINT_B:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 8);
+ case AArch64::ATOMIC_STORE_HINT_H:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 16);
+ case AArch64::ATOMIC_STORE_HINT_S:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 32);
+ case AArch64::ATOMIC_STORE_HINT_D:
+ return expandAtomicStoreHintPseudo(MBB, MBBI, 64);
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index feb8f46583794..331c1354e9854 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -12,6 +12,7 @@
#include "AArch64.h"
#include "AArch64ExpandImm.h"
+#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
@@ -23,6 +24,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
@@ -516,6 +518,10 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
+ bool isAtomicHintInst(SDNode *N, AArch64AtomicStoreHint Hint) const;
+ bool isAtomicSTSHH_KEEP(SDNode *N) const;
+ bool isAtomicSTSHH_STRM(SDNode *N) const;
+
bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
bool Negate);
bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
@@ -4614,6 +4620,20 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
+bool AArch64DAGToDAGISel::isAtomicHintInst(SDNode *N,
+ AArch64AtomicStoreHint Hint) const {
+ const MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
+ return AArch64InstrInfo::decodeAtomicHintFlags(MMO->getFlags()) == Hint;
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
+ return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+}
+
+bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
+ return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+}
+
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
SDValue &Shift, bool Negate) {
if (!isa<ConstantSDNode>(N))
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d90030ee5519e..39fd9a5391d46 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -79,6 +79,7 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
@@ -19409,7 +19410,25 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
I.hasMetadata(FALKOR_STRIDED_ACCESS_MD))
return MOStridedAccess;
- return MachineMemOperand::MONone;
+
+ auto Flags = MachineMemOperand::MONone;
+ const MDNode *AtomicStHint = I.getMetadata(AARCH64_ATOMIC_STORE_HINT_MD);
+ if (AtomicStHint) {
+ unsigned HintVal =
+ cast<ConstantInt>(
+ cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
+ ->getZExtValue();
+ AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
+ assert(Hint != AArch64AtomicStoreHint::HINT_NONE &&
+ "Unrecognised atomic hint value requested.");
+
+ if (static_cast<unsigned>(Hint) & 0b1)
+ Flags |= MOAtomicHintBit0;
+ if (static_cast<unsigned>(Hint) & 0b10)
+ Flags |= MOAtomicHintBit1;
+ }
+
+ return Flags;
}
bool AArch64TargetLowering::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2187f21abb70f..7d719239ecc02 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -282,6 +282,115 @@ def : Pat<(relaxed_store<atomic_store_64>
(am_unscaled64 GPR64sp:$Rn, simm9:$offset), (i64 (bitconvert (f64 FPR64Op:$val)))),
(STURDi FPR64Op:$val, GPR64sp:$Rn, simm9:$offset)>;
+//===----------------------------------
+// Atomic store with hint pseudos
+//===----------------------------------
+
+class seq_cst_store<PatFrags base>
+ : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
+ let IsAtomic = 1;
+ let IsAtomicOrderingSequentiallyConsistent = 1;
+}
+
+class atomic_hint_stshh_keep_relaxed<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (relaxed_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_keep_release<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (releasing_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_keep_seqcst<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (seq_cst_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_KEEP(N); }]>;
+
+class atomic_hint_stshh_strm_relaxed<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (relaxed_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class atomic_hint_stshh_strm_release<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (releasing_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class atomic_hint_stshh_strm_seqcst<PatFrag base>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (seq_cst_store<base> node:$ptr, node:$val),
+ [{ return isAtomicSTSHH_STRM(N); }]>;
+
+class BaseStoreHintPseudo<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
+ let isCodeGenOnly = 1;
+ let hasSideEffects = 1;
+ let mayStore = 1;
+}
+
+def ATOMIC_STORE_HINT_B : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
+def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
+
+let AddedComplexity = 15 in {
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
+ def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 0))>;
+
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 1))>;
+
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 1))>;
+
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
+ def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 1))>;
+}
+
//===----------------------------------
// Low-level exclusive operations
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index b8e74f8ad496d..472d5128372b1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2997,6 +2997,19 @@ bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
});
}
+AArch64AtomicStoreHint
+AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
+ unsigned AtomicHint = 0;
+ if (MMOFlags & MOAtomicHintBit0)
+ AtomicHint += 0b1;
+ if (MMOFlags & MOAtomicHintBit1)
+ AtomicHint += 0b10;
+
+ if (!isValidAArch64AtomicHintValue(AtomicHint))
+ return AArch64AtomicStoreHint::HINT_NONE;
+ return static_cast<AArch64AtomicStoreHint>(AtomicHint);
+}
+
bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
switch (Opc) {
default:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 99e0142824d43..d6d7ffb094360 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -16,6 +16,7 @@
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/TypeSize.h"
#include <optional>
@@ -30,8 +31,13 @@ static const MachineMemOperand::Flags MOSuppressPair =
MachineMemOperand::MOTargetFlag1;
static const MachineMemOperand::Flags MOStridedAccess =
MachineMemOperand::MOTargetFlag2;
+static const MachineMemOperand::Flags MOAtomicHintBit0 =
+ MachineMemOperand::MOTargetFlag3;
+static const MachineMemOperand::Flags MOAtomicHintBit1 =
+ MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
+#define AARCH64_ATOMIC_STORE_HINT_MD "aarch64.atomic.hint"
// AArch64 MachineCombiner patterns
enum AArch64MachineCombinerPattern : unsigned {
@@ -235,6 +241,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
+ static AArch64AtomicStoreHint
+ decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags);
+
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
static bool hasUnscaledLdStOffset(MachineInstr &MI) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index f42e1864d41b5..f4865af72f5d3 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -45,6 +45,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Type.h"
#include "llvm/Pass.h"
+#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
@@ -2547,6 +2548,66 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
I.eraseFromParent();
return true;
}
+ case TargetOpcode::G_STORE: {
+ GStore &St = cast<GStore>(I);
+ auto MMO = St.getMMO();
+ LLT PtrTy = MRI.getType(St.getPointerReg());
+
+ // Only for handling atomic store with hint.
+ // Can only handle AddressSpace 0, 64-bit pointers.
+ if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64)) {
+ return false;
+ }
+
+ AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
+ if (Hint == AArch64AtomicStoreHint::HINT_NONE)
+ return false;
+
+ unsigned HintOpc;
+ unsigned StoreSize = St.getMemSizeInBits().getValue();
+ Register ValueReg = St.getValueReg();
+ switch (StoreSize) {
+ case 8:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_B;
+ break;
+ case 16: {
+ Register CastReg;
+ if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
+ auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
+ {&AArch64::FPR32RegClass}, {});
+ auto Ins = MIB.buildInstr(TargetOpcode::INSERT_SUBREG,
+ {&AArch64::FPR32RegClass}, {Undef, ValueReg})
+ .addImm(AArch64::hsub);
+ constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
+ constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
+ ValueReg = Ins.getReg(0);
+ }
+ HintOpc = AArch64::ATOMIC_STORE_HINT_H;
+ break;
+ }
+ case 32:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_S;
+ break;
+ case 64:
+ HintOpc = AArch64::ATOMIC_STORE_HINT_D;
+ break;
+ default:
+ llvm_unreachable("Unexpected getMemSizeInBits() value for atomic hint.");
+ }
+
+ unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
+
+ auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
+ .addReg(St.getPointerReg())
+ .addReg(ValueReg)
+ .addImm((int)toCABI(St.getMMO().getSuccessOrdering()))
+ .addImm(static_cast<unsigned>(HintImm));
+
+ StrPseudo.cloneMemRefs(I);
+ I.eraseFromParent();
+ constrainSelectedInstRegOperands(*StrPseudo, TII, TRI, RBI);
+ return true;
+ }
default:
return false;
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
new file mode 100644
index 0000000000000..dfcfa92cbc6c8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -0,0 +1,288 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -verify-machineinstrs < %s | FileCheck %s
+
+;
+; STSHH: Keep, Relaxed
+;
+
+define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_relaxed_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, Release
+;
+
+define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Keep, SequentiallyConsistent
+;
+
+define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_seqcst_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; STSHH: Stream, Relaxed
+;
+
+define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; STSHH: Stream, Release
+;
+
+define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i16 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i32 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_release_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x1, [x0]
+; CHECK-NEXT: ret
+ store atomic i64 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; STSHH: Stream, SequentiallyConsistent
+;
+
+define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrh w8, [x0]
+; CHECK-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr w8, [x0]
+; CHECK-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlr x8, [x0]
+; CHECK-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+!0 = !{i32 0}
+!1 = !{i32 1}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
new file mode 100644
index 0000000000000..ed69efafb04c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
@@ -0,0 +1,30 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ define void @test_atomic_store_keep_release_i8(ptr %ptr, i8 %val) {
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ ret void
+ }
+
+ !0 = !{i32 0}
+...
+
+---
+name: test_atomic_store_keep_release_i8
+liveins:
+ - { reg: '$x0', virtual-reg: '' }
+ - { reg: '$w1', virtual-reg: '' }
+body: |
+ bb.0 (%ir-block.0):
+ liveins: $w1, $x0
+
+ ; CHECK-LABEL: name: test_atomic_store_keep_release_i8
+ ; CHECK: BUNDLE implicit killed $w1, implicit $x0 :: (store release (s8) into %ir.ptr, align 8) {
+ ; CHECK-NEXT: STSHH 0
+ ; CHECK-NEXT: STRBBui killed renamable $w1, $x0, 0 :: (store release (s8) into %ir.ptr, align 8)
+ ; CHECK-NEXT: }
+ ; CHECK-NEXT: RET undef $lr
+
+ ATOMIC_STORE_HINT_B killed renamable $x0, killed renamable $w1, 0, 0 :: (store release (s8) into %ir.ptr, align 8)
+ RET_ReallyLR
+...
>From 20d0f7432abe370a81e90a628dc73f102c6430b3 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 19 May 2026 10:43:46 +0000
Subject: [PATCH 02/13] Move expandAtomicStoreHintPseudo to
AArch64AsmPrinter.cpp
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 51 +++++++++++++++++
.../AArch64/AArch64ExpandPseudoInsts.cpp | 56 -------------------
.../lib/Target/AArch64/AArch64InstrAtomics.td | 1 +
.../Atomics/aarch64-atomic-store-hint.mir | 30 ----------
4 files changed, 52 insertions(+), 86 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index bc8c5510ea515..518f0bd99f8b4 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -290,6 +290,9 @@ class AArch64AsmPrinter : public AsmPrinter {
// Emit expansion of Compare-and-branch pseudo instructions
void emitCBPseudoExpansion(const MachineInstr *MI);
+ // Emit expansion of atomic store with hint pseudo instructions
+ void emitAtomicHintPseudoExpansion(const MachineInstr *MI, unsigned Size);
+
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
EmitToStreamer(*OutStreamer, Inst);
@@ -3249,6 +3252,42 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, Inst);
}
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
+ unsigned Size) {
+
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(2).getImm();
+ bool Relaxed = Order == 0;
+ switch (Size) {
+ case 8:
+ StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
+ break;
+ case 16:
+ StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
+ break;
+ case 32:
+ StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
+ break;
+ case 64:
+ StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint size.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(3).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Store.setFlags(MI->getFlags());
+ if (Relaxed)
+ Store.addOperand(MCOperand::createImm(0));
+ EmitToStreamer(*OutStreamer, Store);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -3966,6 +4005,18 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::CBXPrr:
emitCBPseudoExpansion(MI);
return;
+ case AArch64::ATOMIC_STORE_HINT_B:
+ emitAtomicHintPseudoExpansion(MI, 8);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_H:
+ emitAtomicHintPseudoExpansion(MI, 16);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_S:
+ emitAtomicHintPseudoExpansion(MI, 32);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_D:
+ emitAtomicHintPseudoExpansion(MI, 64);
+ return;
}
if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index f07bc127318d5..742721a2df16d 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -68,9 +68,6 @@ class AArch64ExpandPseudoImpl {
MachineBasicBlock::iterator MBBI);
bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
- bool expandAtomicStoreHintPseudo(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- unsigned Size);
bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
unsigned ExtendImm, unsigned ZeroReg,
@@ -1312,51 +1309,6 @@ bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
return true;
}
-bool AArch64ExpandPseudoImpl::expandAtomicStoreHintPseudo(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Size) {
- MachineInstr &MI = *MBBI;
- DebugLoc DL = MI.getDebugLoc();
-
- unsigned StOpc;
- unsigned Order = MI.getOperand(2).getImm();
- bool Relaxed = Order == 0;
-
- switch (Size) {
- case 8:
- StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
- break;
- case 16:
- StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
- break;
- case 32:
- StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
- break;
- case 64:
- StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
- break;
- default:
- llvm_unreachable("Unexpected atomic hint size.");
- }
-
- auto *Hint = BuildMI(MBB, MBBI, DL, TII->get(AArch64::STSHH))
- .addImm(MI.getOperand(3).getImm())
- .getInstr();
-
- auto Store = BuildMI(MBB, MBBI, DL, TII->get(StOpc))
- .add(MI.getOperand(1))
- .addReg(MI.getOperand(0).getReg())
- .setMemRefs(MI.memoperands())
- .setMIFlags(MI.getFlags());
-
- if (Relaxed)
- Store.addImm(0);
-
- transferImpOps(MI, Store, Store);
- finalizeBundle(MBB, Hint->getIterator(), MBBI->getIterator());
- MI.eraseFromParent();
- return true;
-}
-
/// If MBBI references a pseudo instruction that should be expanded here,
/// do the expansion and return true. Otherwise return false.
bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
@@ -2059,14 +2011,6 @@ bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case AArch64::NAND_ZZZ:
case AArch64::NOR_ZZZ:
return expandSVEBitwisePseudo(MI, MBB, MBBI);
- case AArch64::ATOMIC_STORE_HINT_B:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 8);
- case AArch64::ATOMIC_STORE_HINT_H:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 16);
- case AArch64::ATOMIC_STORE_HINT_S:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 32);
- case AArch64::ATOMIC_STORE_HINT_D:
- return expandAtomicStoreHintPseudo(MBB, MBBI, 64);
}
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 7d719239ecc02..af57d471e0411 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -325,6 +325,7 @@ class atomic_hint_stshh_strm_seqcst<PatFrag base>
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
+ let Size = 8;
let isCodeGenOnly = 1;
let hasSideEffects = 1;
let mayStore = 1;
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
deleted file mode 100644
index ed69efafb04c3..0000000000000
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
-
---- |
- define void @test_atomic_store_keep_release_i8(ptr %ptr, i8 %val) {
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
- ret void
- }
-
- !0 = !{i32 0}
-...
-
----
-name: test_atomic_store_keep_release_i8
-liveins:
- - { reg: '$x0', virtual-reg: '' }
- - { reg: '$w1', virtual-reg: '' }
-body: |
- bb.0 (%ir-block.0):
- liveins: $w1, $x0
-
- ; CHECK-LABEL: name: test_atomic_store_keep_release_i8
- ; CHECK: BUNDLE implicit killed $w1, implicit $x0 :: (store release (s8) into %ir.ptr, align 8) {
- ; CHECK-NEXT: STSHH 0
- ; CHECK-NEXT: STRBBui killed renamable $w1, $x0, 0 :: (store release (s8) into %ir.ptr, align 8)
- ; CHECK-NEXT: }
- ; CHECK-NEXT: RET undef $lr
-
- ATOMIC_STORE_HINT_B killed renamable $x0, killed renamable $w1, 0, 0 :: (store release (s8) into %ir.ptr, align 8)
- RET_ReallyLR
-...
>From 2d47e98d9a98b21e3f09bb478070aef290c9fda2 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 12 Jun 2026 10:38:00 +0000
Subject: [PATCH 03/13] - Document aarch64.atomic.hint in LangRef - Rewrite
patterns - Use LLVM ordering in patterns & expansion - Add Clang tests for
more types - Remove AARCH64_ATOMIC_STORE_HINT_MD
---
.../clang/Basic/DiagnosticSemaKinds.td | 10 +-
clang/lib/Sema/SemaARM.cpp | 9 +-
clang/test/CodeGen/arm_acle.c | 90 ++++++++++++++
clang/test/Sema/builtins-arm64.c | 6 +-
llvm/docs/LangRef.md | 26 ++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 2 +-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 115 ++++--------------
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 1 -
.../GISel/AArch64InstructionSelector.cpp | 32 ++++-
10 files changed, 185 insertions(+), 108 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 11a2ccfd1e0b1..9baa201c05239 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9615,6 +9615,11 @@ def err_atomic_builtin_must_be_pointer : Error<
def err_atomic_builtin_must_be_pointer_intptr : Error<
"address argument to atomic builtin must be a pointer to integer or pointer"
" (%0 invalid)">;
+
+def err_atomic_hint_builtin_must_be_pointer : Error<
+ "address argument to atomic hint builtin must be a pointer to a scalar "
+ "integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+
def err_atomic_builtin_cannot_be_const : Error<
"address argument to atomic builtin cannot be const-qualified (%0 invalid)">;
def err_atomic_builtin_must_be_pointer_intfltptr : Error<
@@ -9687,8 +9692,9 @@ def err_atomic_op_hint_data_size : Error<
"address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits">;
def err_atomic_hint_has_invalid_memory_order : Error<
"invalid memory order argument to atomic hint operation (%0 invalid)">;
-def err_atomic_hint_has_invalid_hint_type : Error<
- "invalid hint type argument to atomic hint operation (%0 invalid)">;
+def warn_atomic_hint_has_invalid_hint_type : Warning<
+ "unrecognised hint type argument to atomic hint operation (%0)">,
+ InGroup<DiagGroup<"atomic-hint-type">>;
def warn_atomic_op_has_invalid_memory_order : Warning<
"%select{|success |failure }0memory order argument to atomic operation is invalid">,
InGroup<DiagGroup<"atomic-memory-ordering">>;
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 7a03f25a7e5c9..c4da19526d0a7 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -338,7 +338,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_builtin_must_be_pointer)
+ diag::err_atomic_hint_builtin_must_be_pointer)
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
@@ -348,8 +348,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_op_needs_atomic_int_or_fp)
<< 0 << PtrQT << PtrArg->getSourceRange();
- unsigned TySize =
- Context.getTypeSize(Context.getCanonicalType(PtrQT).getUnqualifiedType());
+ unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
<< PtrArg->getSourceRange();
@@ -398,14 +397,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
if (!HintAP)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_hint_type)
+ diag::warn_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
unsigned Hint = HintAP->getZExtValue();
if (llvm::getAtomicStoreHintFromMD(Hint) ==
llvm::AArch64AtomicStoreHint::HINT_NONE)
return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_hint_type)
+ diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
return false;
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 29e35ed56fa69..1c42a4a171140 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1773,6 +1773,24 @@ void test_atomic_store_hint_char(char *ptr, char data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_uchar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_schar(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_bfloat(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
@@ -1782,6 +1800,15 @@ void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
}
+// AArch64-LABEL: @test_atomic_store_hint_half(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_short(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
@@ -1791,6 +1818,33 @@ void test_atomic_store_hint_short(short *ptr, short data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_ushort(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_int(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_int(int *ptr, int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_unsigned(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_u32(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
@@ -1800,6 +1854,15 @@ void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_s32(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_float(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
@@ -1818,6 +1881,24 @@ void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
}
+// AArch64-LABEL: @test_atomic_store_hint_long(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long(long *ptr, long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_ulong(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
@@ -1827,6 +1908,15 @@ void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_double(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index fb4718a1bd1f4..5372aff24b739 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -58,13 +58,13 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
- __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}}
+ __builtin_arm_atomic_store_with_hint(0, c_data, 0, 0); // expected-error {{address argument to atomic hint builtin must be a pointer to a scalar integral or floating-point type of 8, 16, 32, or 64 bits ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
__builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int' invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-error {{invalid hint type argument to atomic hint operation (3 invalid)}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-warning {{unrecognised hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
}
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 4966065f2207c..3447a9094daef 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -8784,6 +8784,32 @@ of the node are ignored.
This metadata was previously spelled `amdgpu.ignore.denormal.mode`.
+(metadata_aarch64_atomic_hint)=
+
+#### '``aarch64.atomic.hint``' Metadata
+
+The ``aarch64.atomic.hint`` metadata may be attached to an atomic store
+instruction, referencing a single metadata node containing a single ``i32``
+entry:
+
+```llvm
+ store atomic i64 %x, ptr %y seq_cst, align 8, !aarch64.atomic.hint !0
+
+ ...
+ !0 = !{i32 1}
+```
+
+On AArch64 targets, this metadata may be used to emit an atomic store together
+with a hint instruction. The hint is a suggestion to the compiler which may be
+used when selecting code sequences, but it is not required to emit a specific
+hint instruction. The following hint values are currently recognised:
+
+- ``0``: ``stshh keep`` hint.
+- ``1``: ``stshh strm`` hint.
+
+If the compiler does not recognise the hint value provided, it may ignore the
+metadata. Targets that do not support this metadata may also ignore it.
+
#### '`type`' Metadata
See {doc}`TypeMetadata`.
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 518f0bd99f8b4..7d194f215d936 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3257,7 +3257,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
unsigned StOpc;
unsigned Order = MI->getOperand(2).getImm();
- bool Relaxed = Order == 0;
+ bool Relaxed = Order == 2;
switch (Size) {
case 8:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 39fd9a5391d46..586413752069c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19412,7 +19412,7 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
return MOStridedAccess;
auto Flags = MachineMemOperand::MONone;
- const MDNode *AtomicStHint = I.getMetadata(AARCH64_ATOMIC_STORE_HINT_MD);
+ const MDNode *AtomicStHint = I.getMetadata("aarch64.atomic.hint");
if (AtomicStHint) {
unsigned HintVal =
cast<ConstantInt>(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index af57d471e0411..af06ef9014031 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -286,42 +286,6 @@ def : Pat<(relaxed_store<atomic_store_64>
// Atomic store with hint pseudos
//===----------------------------------
-class seq_cst_store<PatFrags base>
- : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
- let IsAtomic = 1;
- let IsAtomicOrderingSequentiallyConsistent = 1;
-}
-
-class atomic_hint_stshh_keep_relaxed<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (relaxed_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_keep_release<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (releasing_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_keep_seqcst<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (seq_cst_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_KEEP(N); }]>;
-
-class atomic_hint_stshh_strm_relaxed<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (relaxed_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
-class atomic_hint_stshh_strm_release<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (releasing_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
-class atomic_hint_stshh_strm_seqcst<PatFrag base>
- : PatFrag<(ops node:$ptr, node:$val),
- (seq_cst_store<base> node:$ptr, node:$val),
- [{ return isAtomicSTSHH_STRM(N); }]>;
-
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
@@ -336,60 +300,33 @@ def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
+class atomic_hint_store<PatFrag Base, bit Rel, bit SeqCst, code Pred, code GIPred>
+ : PatFrag<(ops node:$ptr, node:$val),
+ (Base node:$val, node:$ptr), Pred> {
+ let IsAtomic = 1;
+ let IsAtomicOrderingReleaseOrStronger = Rel;
+ let IsAtomicOrderingSequentiallyConsistent = SeqCst;
+ let GISelPredicateCode = GIPred;
+}
+
+multiclass AtomicHintPatterns<int Order, int Hint, bit Rel, bit SeqCst, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR64:$data),
+ (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
+}
+
let AddedComplexity = 15 in {
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 0))>;
- def : Pat<(atomic_hint_stshh_keep_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 0))>;
-
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 0), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_relaxed<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 0), (i32 1))>;
-
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 3), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_release<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 3), (i32 1))>;
-
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_8> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_16> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_32> GPR64sp:$addr, GPR32:$data),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 5), (i32 1))>;
- def : Pat<(atomic_hint_stshh_strm_seqcst<atomic_store_64> GPR64sp:$addr, GPR64:$data),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 5), (i32 1))>;
+ defm : AtomicHintPatterns<2, 0, 0, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 0, 1, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 0, 0, 1, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 1, 0, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 1, 1, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 1, 0, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index d6d7ffb094360..7bfaf4471b8c0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -37,7 +37,6 @@ static const MachineMemOperand::Flags MOAtomicHintBit1 =
MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
-#define AARCH64_ATOMIC_STORE_HINT_MD "aarch64.atomic.hint"
// AArch64 MachineCombiner patterns
enum AArch64MachineCombinerPattern : unsigned {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index f4865af72f5d3..0e2ea50df5e81 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -530,6 +530,11 @@ class AArch64InstructionSelector : public InstructionSelector {
MachineOperand &Predicate,
MachineIRBuilder &MIRBuilder) const;
+ bool isAtomicHintInst(const MachineInstr &MI,
+ AArch64AtomicStoreHint Hint) const;
+ bool isAtomicSTSHH_KEEP(const MachineInstr &MI) const;
+ bool isAtomicSTSHH_STRM(const MachineInstr &MI) const;
+
/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
@@ -2552,17 +2557,15 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
GStore &St = cast<GStore>(I);
auto MMO = St.getMMO();
LLT PtrTy = MRI.getType(St.getPointerReg());
+ AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
// Only for handling atomic store with hint.
// Can only handle AddressSpace 0, 64-bit pointers.
- if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64)) {
+ if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64) ||
+ Hint == AArch64AtomicStoreHint::HINT_NONE) {
return false;
}
- AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
- if (Hint == AArch64AtomicStoreHint::HINT_NONE)
- return false;
-
unsigned HintOpc;
unsigned StoreSize = St.getMemSizeInBits().getValue();
Register ValueReg = St.getValueReg();
@@ -2600,7 +2603,7 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
.addReg(St.getPointerReg())
.addReg(ValueReg)
- .addImm((int)toCABI(St.getMMO().getSuccessOrdering()))
+ .addImm((int)MMO.getSuccessOrdering())
.addImm(static_cast<unsigned>(HintImm));
StrPseudo.cloneMemRefs(I);
@@ -8396,6 +8399,23 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
.getZExtValue()));
}
+bool AArch64InstructionSelector::isAtomicHintInst(
+ const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
+ const GStore &St = cast<GStore>(MI);
+ auto MMO = St.getMMO();
+ return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
+}
+
+bool AArch64InstructionSelector::isAtomicSTSHH_KEEP(
+ const MachineInstr &MI) const {
+ return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+}
+
+bool AArch64InstructionSelector::isAtomicSTSHH_STRM(
+ const MachineInstr &MI) const {
+ return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+}
+
bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
>From e9a4296b30d1718f703b6b1463fed5bdf36fbdec Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 23 Jun 2026 10:28:36 +0000
Subject: [PATCH 04/13] - Add support for different addressing modes when the
ordering is relaxed - Set IsAtomicOrderingMonotonic & IsAtomicOrderingRelease
in atomic_hint_store - Do not attach invalid hints to atomic_store & ignore
any invalid hints found
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 18 +-
clang/lib/Sema/SemaARM.cpp | 19 +-
clang/test/CodeGen/builtins-arm64.c | 4 +
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 143 +++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 2 -
.../lib/Target/AArch64/AArch64InstrAtomics.td | 137 ++++--
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 -
.../GISel/AArch64InstructionSelector.cpp | 92 ++--
.../Atomics/aarch64-atomic-store-hint.ll | 62 ++-
.../Atomics/aarch64-relaxed-store-hint.ll | 416 ++++++++++++++++++
10 files changed, 780 insertions(+), 115 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index efe742e3bc091..e53866be4af59 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2103,15 +2103,15 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
llvm_unreachable(
"Expected integer hint argument to atomic store with hint.");
unsigned HintArg = Result.Val.getInt().getExtValue();
- assert((getAtomicStoreHintFromMD(HintArg) !=
- AArch64AtomicStoreHint::HINT_NONE) &&
- "Invalid hint type");
-
- MDNode *HintMDVal =
- MDNode::get(CGM.getLLVMContext(),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
- Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
- HintMDVal);
+
+ // Attach the hint if valid
+ if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
+ MDNode *HintMDVal =
+ MDNode::get(CGM.getLLVMContext(),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
+ Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
+ HintMDVal);
+ }
return Store;
}
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index c4da19526d0a7..95f4b4237f1e8 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -331,10 +331,11 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// Arg 0 should be the pointer type. The pointee type must be a
// scalar integral or floating-point type of 8, 16, 32 or 64 bits.
ASTContext &Context = getASTContext();
- Expr *PtrArg = TheCall->getArg(0);
- auto PtrArgRes = SemaRef.DefaultFunctionArrayLvalueConversion(PtrArg);
+ auto PtrArgRes =
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(0));
if (PtrArgRes.isInvalid())
return true;
+ auto *PtrArg = PtrArgRes.get();
auto *PtrTy = PtrArg->getType()->getAs<PointerType>();
if (!PtrTy)
return Diag(TheCall->getBeginLoc(),
@@ -395,17 +396,17 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP)
- return Diag(TheCall->getBeginLoc(),
- diag::warn_atomic_hint_has_invalid_hint_type)
- << HintArg->getType() << HintArg->getSourceRange();
+ if (!HintAP) {
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << HintArg->getType() << HintArg->getSourceRange();
+ return false;
+ }
unsigned Hint = HintAP->getZExtValue();
if (llvm::getAtomicStoreHintFromMD(Hint) ==
llvm::AArch64AtomicStoreHint::HINT_NONE)
- return Diag(TheCall->getBeginLoc(),
- diag::warn_atomic_hint_has_invalid_hint_type)
- << *HintAP << HintArg->getSourceRange();
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << *HintAP << HintArg->getSourceRange();
return false;
}
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 74553659fbfed..7d3b83bef3179 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -225,6 +225,10 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+
+ // Invalid hint should be dropped
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 7d194f215d936..b467032c6d0ce 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -291,7 +291,9 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitCBPseudoExpansion(const MachineInstr *MI);
// Emit expansion of atomic store with hint pseudo instructions
- void emitAtomicHintPseudoExpansion(const MachineInstr *MI, unsigned Size);
+ void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
+ void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
void EmitToStreamer(const MCInst &Inst) {
@@ -3252,23 +3254,22 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, Inst);
}
-void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
- unsigned Size) {
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
unsigned StOpc;
unsigned Order = MI->getOperand(2).getImm();
bool Relaxed = Order == 2;
- switch (Size) {
- case 8:
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_B:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
break;
- case 16:
+ case AArch64::ATOMIC_STORE_HINT_H:
StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
break;
- case 32:
+ case AArch64::ATOMIC_STORE_HINT_S:
StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
break;
- case 64:
+ case AArch64::ATOMIC_STORE_HINT_D:
StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
break;
default:
@@ -3288,6 +3289,104 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI,
EmitToStreamer(*OutStreamer, Store);
}
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(5).getImm();
+ assert(Order == 2 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ StOpc = AArch64::STRBBroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ StOpc = AArch64::STRHHroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ StOpc = AArch64::STRWroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ StOpc = AArch64::STRXroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ StOpc = AArch64::STRBBroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ StOpc = AArch64::STRHHroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ StOpc = AArch64::STRWroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ StOpc = AArch64::STRXroX;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(6).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm
+ Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed
+ Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
+void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
+ const MachineInstr *MI) {
+ unsigned StOpc;
+ unsigned Order = MI->getOperand(3).getImm();
+ assert(Order == 2 &&
+ "Atomic store addressing mode only supports relaxed stores");
+
+ switch (MI->getOpcode()) {
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ StOpc = AArch64::STRBBui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ StOpc = AArch64::STRHHui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ StOpc = AArch64::STRWui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ StOpc = AArch64::STRXui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ StOpc = AArch64::STURBBi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ StOpc = AArch64::STURHHi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ StOpc = AArch64::STURWi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ StOpc = AArch64::STURXi;
+ break;
+ default:
+ llvm_unreachable("Unexpected atomic hint opcode.");
+ }
+
+ EmitToStreamer(
+ MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(4).getImm()));
+
+ MCInst Store;
+ Store.setOpcode(StOpc);
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
+ Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm
+ Store.setFlags(MI->getFlags());
+ EmitToStreamer(*OutStreamer, Store);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -4006,16 +4105,30 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
emitCBPseudoExpansion(MI);
return;
case AArch64::ATOMIC_STORE_HINT_B:
- emitAtomicHintPseudoExpansion(MI, 8);
- return;
case AArch64::ATOMIC_STORE_HINT_H:
- emitAtomicHintPseudoExpansion(MI, 16);
- return;
case AArch64::ATOMIC_STORE_HINT_S:
- emitAtomicHintPseudoExpansion(MI, 32);
- return;
case AArch64::ATOMIC_STORE_HINT_D:
- emitAtomicHintPseudoExpansion(MI, 64);
+ emitAtomicHintPseudoExpansion(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_BroW:
+ case AArch64::ATOMIC_STORE_HINT_HroW:
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ case AArch64::ATOMIC_STORE_HINT_BroX:
+ case AArch64::ATOMIC_STORE_HINT_HroX:
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ emitAtomicHintPseudoExpansionRO(MI);
+ return;
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ emitAtomicHintPseudoExpansionImm(MI);
return;
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 586413752069c..20c221a7a5d24 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19419,8 +19419,6 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
->getZExtValue();
AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
- assert(Hint != AArch64AtomicStoreHint::HINT_NONE &&
- "Unrecognised atomic hint value requested.");
if (static_cast<unsigned>(Hint) & 0b1)
Flags |= MOAtomicHintBit0;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index af06ef9014031..e0dc12bc62de6 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -286,47 +286,126 @@ def : Pat<(relaxed_store<atomic_store_64>
// Atomic store with hint pseudos
//===----------------------------------
-class BaseStoreHintPseudo<RegisterClass regtype>
- : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]> {
- let Size = 8;
- let isCodeGenOnly = 1;
- let hasSideEffects = 1;
- let mayStore = 1;
-}
-
-def ATOMIC_STORE_HINT_B : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_H : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_S : BaseStoreHintPseudo<GPR32>;
-def ATOMIC_STORE_HINT_D : BaseStoreHintPseudo<GPR64>;
-
-class atomic_hint_store<PatFrag Base, bit Rel, bit SeqCst, code Pred, code GIPred>
- : PatFrag<(ops node:$ptr, node:$val),
+let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
+ class BaseStoreHintPseudo<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, GPR32:$Rm, regtype:$data, WExt:$extend,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, GPR64:$Rm, regtype:$data, XExt:$extend,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, UImmOp:$offset,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+
+ class BaseStoreHintPseudoImm<RegisterClass regtype>
+ : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, simm9:$offset,
+ i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+}
+
+multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> {
+ def NAME : BaseStoreHintPseudo<regtype>;
+ def NAME # roW : BaseStoreHintPseudoROW<regtype, WExt>;
+ def NAME # roX : BaseStoreHintPseudoROX<regtype, XExt>;
+ def NAME # ui : BaseStoreHintPseudoUImm<regtype, UImmOp>;
+ def NAME # i : BaseStoreHintPseudoImm<regtype>;
+}
+
+defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>;
+defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
+defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+
+class atomic_hint_store<PatFrag Base, int Order, code Pred, code GIPred>
+ : PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
- let IsAtomicOrderingReleaseOrStronger = Rel;
- let IsAtomicOrderingSequentiallyConsistent = SeqCst;
+ let IsAtomicOrderingMonotonic = !eq(Order, 2);
+ let IsAtomicOrderingRelease = !eq(Order, 5);
+ let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
let GISelPredicateCode = GIPred;
}
-multiclass AtomicHintPatterns<int Order, int Hint, bit Rel, bit SeqCst, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+multiclass AtomicHintPatternsAddrMode<int Hint, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
+ (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
+ (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
+ (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
+ (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
+}
+
+multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR32:$data),
+ def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Rel, SeqCst, Pred, GIPred> GPR64sp:$addr, GPR64:$data),
+ def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred, GIPred> GPR64:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatterns<2, 0, 0, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 0, 1, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 0, 0, 1, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 1, 0, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 1, 1, 0, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 1, 0, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+
+ defm : AtomicHintPatterns<2, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 472d5128372b1..facff777d9012 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3005,8 +3005,6 @@ AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
if (MMOFlags & MOAtomicHintBit1)
AtomicHint += 0b10;
- if (!isValidAArch64AtomicHintValue(AtomicHint))
- return AArch64AtomicStoreHint::HINT_NONE;
return static_cast<AArch64AtomicStoreHint>(AtomicHint);
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 0e2ea50df5e81..4ddf563e14633 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2566,14 +2566,11 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
return false;
}
- unsigned HintOpc;
- unsigned StoreSize = St.getMemSizeInBits().getValue();
+ // AtomicExpandPass converts FP stores to integer stores of the equivalent
+ // bitwidth. Widen the register here if the original type was f16.
+ unsigned StoreSize = St.getMemSize().getValue();
Register ValueReg = St.getValueReg();
- switch (StoreSize) {
- case 8:
- HintOpc = AArch64::ATOMIC_STORE_HINT_B;
- break;
- case 16: {
+ if (StoreSize == 2) {
Register CastReg;
if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
@@ -2585,26 +2582,72 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
ValueReg = Ins.getReg(0);
}
- HintOpc = AArch64::ATOMIC_STORE_HINT_H;
- break;
}
- case 32:
- HintOpc = AArch64::ATOMIC_STORE_HINT_S;
- break;
- case 64:
- HintOpc = AArch64::ATOMIC_STORE_HINT_D;
- break;
- default:
- llvm_unreachable("Unexpected getMemSizeInBits() value for atomic hint.");
+
+ static constexpr unsigned BaseOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_B, AArch64::ATOMIC_STORE_HINT_H,
+ AArch64::ATOMIC_STORE_HINT_S, AArch64::ATOMIC_STORE_HINT_D};
+ static constexpr unsigned RegWOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_BroW, AArch64::ATOMIC_STORE_HINT_HroW,
+ AArch64::ATOMIC_STORE_HINT_SroW, AArch64::ATOMIC_STORE_HINT_DroW};
+ static constexpr unsigned RegXOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_BroX, AArch64::ATOMIC_STORE_HINT_HroX,
+ AArch64::ATOMIC_STORE_HINT_SroX, AArch64::ATOMIC_STORE_HINT_DroX};
+ static constexpr unsigned UImmOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_Bui, AArch64::ATOMIC_STORE_HINT_Hui,
+ AArch64::ATOMIC_STORE_HINT_Sui, AArch64::ATOMIC_STORE_HINT_Dui};
+ static constexpr unsigned ImmOpcodes[] = {
+ AArch64::ATOMIC_STORE_HINT_Bi, AArch64::ATOMIC_STORE_HINT_Hi,
+ AArch64::ATOMIC_STORE_HINT_Si, AArch64::ATOMIC_STORE_HINT_Di};
+
+ AtomicOrdering Ordering = MMO.getSuccessOrdering();
+ MachineInstrBuilder StrPseudo;
+ unsigned HintOpc = 0;
+
+ // Other addressing modes can be used when the ordering is monotonic.
+ // Try to match these first, before falling back on the basic operands.
+ auto AddModeWRO = selectAddrModeWRO(St.getOperand(1), StoreSize);
+ auto AddModeXRO = selectAddrModeXRO(St.getOperand(1), StoreSize);
+ auto AddModeUImm = selectAddrModeIndexed(St.getOperand(1), StoreSize);
+ auto AddModeImm = selectAddrModeUnscaled(St.getOperand(1), StoreSize);
+
+ if (AddModeWRO && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = RegWOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeWRO)[0](StrPseudo);
+ (*AddModeWRO)[1](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeWRO)[2](StrPseudo);
+ } else if (AddModeXRO && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = RegXOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeXRO)[0](StrPseudo);
+ (*AddModeXRO)[1](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeXRO)[2](StrPseudo);
+ } else if (AddModeUImm && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = UImmOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeUImm)[0](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeUImm)[1](StrPseudo);
+ } else if (AddModeImm && Ordering == AtomicOrdering::Monotonic) {
+ HintOpc = ImmOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
+ (*AddModeImm)[0](StrPseudo);
+ StrPseudo.addReg(ValueReg);
+ (*AddModeImm)[1](StrPseudo);
+ } else {
+ HintOpc = BaseOpcodes[Log2_32(StoreSize)];
+ StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
+ .addReg(St.getPointerReg())
+ .addReg(ValueReg);
}
+ // Add the ordering and hint operands, before erasing the store.
unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
-
- auto StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
- .addReg(St.getPointerReg())
- .addReg(ValueReg)
- .addImm((int)MMO.getSuccessOrdering())
- .addImm(static_cast<unsigned>(HintImm));
+ StrPseudo.addImm((int)Ordering);
+ StrPseudo.addImm(HintImm);
StrPseudo.cloneMemRefs(I);
I.eraseFromParent();
@@ -8401,8 +8444,7 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
bool AArch64InstructionSelector::isAtomicHintInst(
const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
- const GStore &St = cast<GStore>(MI);
- auto MMO = St.getMMO();
+ auto MMO = cast<GStore>(MI).getMMO();
return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index dfcfa92cbc6c8..21aec7dae322f 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -6,7 +6,7 @@
; STSHH: Keep, Relaxed
;
-define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -16,7 +16,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) noun
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -26,7 +26,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) no
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -36,7 +36,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) no
ret void
}
-define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_relaxed_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -50,7 +50,7 @@ define dso_local void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) no
; STSHH: Keep, Release
;
-define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -62,7 +62,7 @@ define dso_local void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %v
ret void
}
-define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_half:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -74,7 +74,7 @@ define dso_local void @test_atomic_store_keep_release_half(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_float:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -85,7 +85,7 @@ define dso_local void @test_atomic_store_keep_release_float(ptr %ptr, float %val
ret void
}
-define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -100,7 +100,7 @@ define dso_local void @test_atomic_store_keep_release_double(ptr %ptr, double %v
; STSHH: Keep, SequentiallyConsistent
;
-define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -110,7 +110,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounw
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -120,7 +120,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nou
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -130,7 +130,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nou
ret void
}
-define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_seqcst_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh keep
@@ -144,7 +144,7 @@ define dso_local void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nou
; STSHH: Stream, Relaxed
;
-define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -156,7 +156,7 @@ define dso_local void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %v
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -168,7 +168,7 @@ define dso_local void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -179,7 +179,7 @@ define dso_local void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val
ret void
}
-define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -194,7 +194,7 @@ define dso_local void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %v
; STSHH: Stream, Release
;
-define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
+define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -204,7 +204,7 @@ define dso_local void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) no
ret void
}
-define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
+define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -214,7 +214,7 @@ define dso_local void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val)
ret void
}
-define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
+define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -224,7 +224,7 @@ define dso_local void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val)
ret void
}
-define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
+define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_release_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: stshh strm
@@ -238,7 +238,7 @@ define dso_local void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val)
; STSHH: Stream, SequentiallyConsistent
;
-define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
+define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -250,7 +250,7 @@ define dso_local void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
+define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
@@ -262,7 +262,7 @@ define dso_local void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val)
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
+define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov w8, s0
@@ -273,7 +273,7 @@ define dso_local void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val)
ret void
}
-define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
+define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_double:
; CHECK: // %bb.0:
; CHECK-NEXT: fmov x8, d0
@@ -284,5 +284,19 @@ define dso_local void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %
ret void
}
+;
+; Invalid Hint
+;
+
+define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: test_atomic_store_invalid_hint:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stlrb w1, [x0]
+; CHECK-NEXT: ret
+ store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !2
+ ret void
+}
+
!0 = !{i32 0}
!1 = !{i32 1}
+!2 = !{i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
new file mode 100644
index 0000000000000..d80d9460c107c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -0,0 +1,416 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -global-isel=1 < %s | FileCheck %s
+
+;
+; W register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w2, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w2, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w2, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x2, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+;
+; X register offset
+;
+
+; Int
+
+define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w2, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 %offset
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w2, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i64 %offset
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w2, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i64 %offset
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x2, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i64 %offset
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i64 %offset
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i64 %offset
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w8, [x0, x1, lsl #2]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i64 %offset
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x8, [x0, x1, lsl #3]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i64 %offset
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; Unsigned immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w1, [x0, #4095]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 4095
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strh w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 2048
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 1024
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x1, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 512
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 2048
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 2048
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str w8, [x0, #8192]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 2048
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: str x8, [x0, #16384]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 2048
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+;
+; Signed immediate offset
+;
+
+; Int
+
+define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturb w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i32 -256
+ store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: sturh w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i16, ptr %ptr, i32 -128
+ store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur w1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i32, ptr %ptr, i32 -64
+ store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stur x1, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr i64, ptr %ptr, i32 -32
+ store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ ret void
+}
+
+; FP
+
+define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr bfloat, ptr %ptr, i32 -128
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: ret
+ %addr = getelementptr half, ptr %ptr, i32 -128
+ store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: sub x8, x0, #512
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str w9, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr float, ptr %ptr, i32 -128
+ store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmov x9, d0
+; CHECK-NEXT: sub x8, x0, #1024
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: str x9, [x8]
+; CHECK-NEXT: ret
+ %addr = getelementptr double, ptr %ptr, i32 -128
+ store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ ret void
+}
+
+!0 = !{ i32 0 }
+!1 = !{ i32 1 }
>From 330626599896ffff8c43c75ce287237d9f9586e4 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 12:55:03 +0000
Subject: [PATCH 05/13] - Replace STSHH with HINT after rebase
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 6 +++---
llvm/lib/Target/AArch64/AArch64InstrAtomics.td | 18 +++++++++---------
.../GISel/AArch64InstructionSelector.cpp | 2 +-
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index b467032c6d0ce..426805cb5d42f 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3277,7 +3277,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(3).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(3).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
@@ -3326,7 +3326,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(6).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(6).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
@@ -3376,7 +3376,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
}
EmitToStreamer(
- MCInstBuilder(AArch64::STSHH).addImm(MI->getOperand(4).getImm()));
+ MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
MCInst Store;
Store.setOpcode(StOpc);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index e0dc12bc62de6..8ab0c13e41edc 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -397,15 +397,15 @@ multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatternsAddrMode<0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatternsAddrMode<1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
-
- defm : AtomicHintPatterns<2, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 0, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 1, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+
+ defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
+ defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 4ddf563e14633..c45f1cca2a5a6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2645,7 +2645,7 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
}
// Add the ordering and hint operands, before erasing the store.
- unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 0 : 1;
+ unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 48 : 49;
StrPseudo.addImm((int)Ordering);
StrPseudo.addImm(HintImm);
>From 7a99f400ac657fee3897091d1d3ed7078680bc11 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 15 Jul 2026 13:26:06 +0000
Subject: [PATCH 06/13] - Remove aarch64.atomic.hint & use mem.cache_hint -
Remove GISel support for now and added atomic stores with the hint to
fallBackToDAGISel - Add mfp8 support in Clang + tests
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 14 +-
clang/lib/Sema/SemaARM.cpp | 4 +-
clang/test/CodeGen/arm_acle.c | 52 +++++---
clang/test/CodeGen/builtins-arm64.c | 12 +-
llvm/docs/LangRef.md | 26 ----
llvm/lib/CodeGen/AtomicExpandPass.cpp | 3 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 4 +-
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 32 ++++-
.../Target/AArch64/AArch64ISelLowering.cpp | 28 ++--
.../lib/Target/AArch64/AArch64InstrAtomics.td | 68 +++++-----
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 12 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 7 +-
.../GISel/AArch64InstructionSelector.cpp | 122 -----------------
.../Atomics/aarch64-atomic-store-hint.ll | 97 ++++++++++----
.../Atomics/aarch64-relaxed-store-hint.ll | 124 +++++++++++++-----
15 files changed, 282 insertions(+), 323 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index e53866be4af59..8a4f19e4e016d 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2106,11 +2106,15 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
// Attach the hint if valid
if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
- MDNode *HintMDVal =
- MDNode::get(CGM.getLLVMContext(),
- llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg)));
- Store->setMetadata(CGM.getModule().getMDKindID("aarch64.atomic.hint"),
- HintMDVal);
+ LLVMContext &Ctx = CGM.getLLVMContext();
+ MDNode *AtomicHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.atomic_hint"),
+ llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
+ MDNode *HintNode = MDNode::get(
+ CGM.getLLVMContext(),
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), AtomicHint});
+
+ Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
}
return Store;
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 95f4b4237f1e8..b3c139bf1c8bc 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -343,8 +343,8 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
- // TODO: Allow MFloat8 types when supported by atomic store
- if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType())
+ if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType() &&
+ !PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_op_needs_atomic_int_or_fp)
<< 0 << PtrQT << PtrArg->getSourceRange();
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 1c42a4a171140..041530a5d2269 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1766,7 +1766,7 @@ int test_rndrrs(uint64_t *__addr) {
// AArch64-LABEL: @test_atomic_store_hint_char(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3:![0-9]+]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1:![0-9]+]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_char(char *ptr, char data) {
@@ -1775,7 +1775,7 @@ void test_atomic_store_hint_char(char *ptr, char data) {
// AArch64-LABEL: @test_atomic_store_hint_uchar(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
@@ -1784,16 +1784,25 @@ void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
// AArch64-LABEL: @test_atomic_store_hint_schar(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i8 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_mfloat(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic <1 x i8> [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_mfloat(__mfp8 *ptr, __mfp8 data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_bfloat(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: store atomic bfloat [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3:![0-9]+]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
@@ -1802,7 +1811,7 @@ void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
// AArch64-LABEL: @test_atomic_store_hint_half(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META4:![0-9]+]]
+// AArch64-NEXT: store atomic half [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
@@ -1811,7 +1820,7 @@ void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
// AArch64-LABEL: @test_atomic_store_hint_short(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_short(short *ptr, short data) {
@@ -1820,7 +1829,7 @@ void test_atomic_store_hint_short(short *ptr, short data) {
// AArch64-LABEL: @test_atomic_store_hint_ushort(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i16 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 2, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
@@ -1829,7 +1838,7 @@ void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
// AArch64-LABEL: @test_atomic_store_hint_int(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_int(int *ptr, int data) {
@@ -1838,7 +1847,7 @@ void test_atomic_store_hint_int(int *ptr, int data) {
// AArch64-LABEL: @test_atomic_store_hint_unsigned(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
@@ -1847,7 +1856,7 @@ void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
// AArch64-LABEL: @test_atomic_store_hint_u32(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
@@ -1856,7 +1865,7 @@ void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
// AArch64-LABEL: @test_atomic_store_hint_s32(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i32 [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
@@ -1865,7 +1874,7 @@ void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
// AArch64-LABEL: @test_atomic_store_hint_float(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic float [[DATA:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_float(float *ptr, float data) {
@@ -1874,7 +1883,7 @@ void test_atomic_store_hint_float(float *ptr, float data) {
// AArch64-LABEL: @test_atomic_store_hint_s64(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
@@ -1883,7 +1892,7 @@ void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
// AArch64-LABEL: @test_atomic_store_hint_long(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long(long *ptr, long data) {
@@ -1892,7 +1901,7 @@ void test_atomic_store_hint_long(long *ptr, long data) {
// AArch64-LABEL: @test_atomic_store_hint_ulong(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
@@ -1901,7 +1910,7 @@ void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
@@ -1910,7 +1919,7 @@ void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data
// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !aarch64.atomic.hint [[META3]]
+// AArch64-NEXT: store atomic i64 [[DATA:%.*]], ptr [[PTR:%.*]] release, align 8, !mem.cache_hint [[HINT1]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
@@ -1919,13 +1928,16 @@ void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned
// AArch64-LABEL: @test_atomic_store_hint_double(
// AArch64-NEXT: entry:
-// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !aarch64.atomic.hint [[META4]]
+// AArch64-NEXT: store atomic double [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 8, !mem.cache_hint [[HINT3]]
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_double(double *ptr, double data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
}
-// AArch64: [[META3]] = !{i32 0}
-// AArch64-NEXT: [[META4]] = !{i32 1}
+// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
+// AArch64-NEXT: [[HINT2]] = !{!"aarch64.atomic_hint", i32 0}
+
+// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
+// AArch64-NEXT: [[HINT4]] = !{!"aarch64.atomic_hint", i32 1}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 7d3b83bef3179..6e11bc652ae9d 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -218,13 +218,13 @@ void trap() {
void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !aarch64.atomic.hint ![[M1:[0-9]]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M1:[0-9]]]
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, 0);
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !aarch64.atomic.hint ![[M1]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M1]]
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
- // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !aarch64.atomic.hint ![[M2:[0-9]]]
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
@@ -232,5 +232,7 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
}
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
-// CHECK: ![[M1]] = !{i32 0}
-// CHECK: ![[M2]] = !{i32 1}
+// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
+// CHECK: ![[M2]] = !{!"aarch64.atomic_hint", i32 0}
+// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
+// CHECK: ![[M4]] = !{!"aarch64.atomic_hint", i32 1}
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 3447a9094daef..4966065f2207c 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -8784,32 +8784,6 @@ of the node are ignored.
This metadata was previously spelled `amdgpu.ignore.denormal.mode`.
-(metadata_aarch64_atomic_hint)=
-
-#### '``aarch64.atomic.hint``' Metadata
-
-The ``aarch64.atomic.hint`` metadata may be attached to an atomic store
-instruction, referencing a single metadata node containing a single ``i32``
-entry:
-
-```llvm
- store atomic i64 %x, ptr %y seq_cst, align 8, !aarch64.atomic.hint !0
-
- ...
- !0 = !{i32 1}
-```
-
-On AArch64 targets, this metadata may be used to emit an atomic store together
-with a hint instruction. The hint is a suggestion to the compiler which may be
-used when selecting code sequences, but it is not required to emit a specific
-hint instruction. The following hint values are currently recognised:
-
-- ``0``: ``stshh keep`` hint.
-- ``1``: ``stshh strm`` hint.
-
-If the compiler does not recognise the hint value provided, it may ignore the
-metadata. Targets that do not support this metadata may also ignore it.
-
#### '`type`' Metadata
See {doc}`TypeMetadata`.
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index d93d01be94709..b3d186fdf5f01 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -246,6 +246,7 @@ static void copyMetadataForAtomic(Instruction &Dest,
case LLVMContext::MD_tbaa:
case LLVMContext::MD_tbaa_struct:
case LLVMContext::MD_alias_scope:
+ case LLVMContext::MD_mem_cache_hint:
case LLVMContext::MD_noalias:
case LLVMContext::MD_noalias_addrspace:
case LLVMContext::MD_access_group:
@@ -257,8 +258,6 @@ static void copyMetadataForAtomic(Instruction &Dest,
Dest.setMetadata(ID, N);
else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))
Dest.setMetadata(ID, N);
- else if (ID == Ctx.getMDKindID("aarch64.atomic.hint"))
- Dest.setMetadata(ID, N);
// Losing atomic.ignore.denormal.mode, but it doesn't matter for current
// uses.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7d185031d7d37..b4cd198683fa0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5454,10 +5454,10 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
report_fatal_error("Cannot generate unaligned atomic store");
auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
-
- MachineFunction &MF = DAG.getMachineFunction();
const MDNode *MemCacheHint =
getMemCacheHintMetadata(I, I.getPointerOperandIndex());
+
+ MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 331c1354e9854..b942b4756c704 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -518,7 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
- bool isAtomicHintInst(SDNode *N, AArch64AtomicStoreHint Hint) const;
+ AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand *MMO) const;
bool isAtomicSTSHH_KEEP(SDNode *N) const;
bool isAtomicSTSHH_STRM(SDNode *N) const;
@@ -4620,18 +4620,36 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
-bool AArch64DAGToDAGISel::isAtomicHintInst(SDNode *N,
- AArch64AtomicStoreHint Hint) const {
- const MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
- return AArch64InstrInfo::decodeAtomicHintFlags(MMO->getFlags()) == Hint;
+AArch64AtomicStoreHint
+AArch64DAGToDAGISel::decodeAtomicHintFlags(MachineMemOperand *MMO) const {
+ int AtomicHint = -1;
+ const MDNode *MemCacheHint = MMO->getMemCacheHint();
+ if (!MemCacheHint)
+ return AArch64AtomicStoreHint::HINT_NONE;
+
+ for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
+ const auto *Key = cast<MDString>(MemCacheHint->getOperand(I));
+ StringRef KeyStr = Key->getString();
+ const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
+
+ if (KeyStr != "aarch64.atomic_hint")
+ continue;
+
+ AtomicHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
+ ->getZExtValue();
+ }
+
+ return getAtomicStoreHintFromMD(AtomicHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
- return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
+ return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64AtomicStoreHint::HINT_STSHH_KEEP;
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
- return isAtomicHintInst(N, AArch64AtomicStoreHint::HINT_STSHH_STRM);
+ return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64AtomicStoreHint::HINT_STSHH_STRM;
}
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 20c221a7a5d24..d5135b47f4ef1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -41,6 +41,7 @@
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
@@ -79,7 +80,6 @@
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
-#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
@@ -19410,23 +19410,7 @@ AArch64TargetLowering::getTargetMMOFlags(const Instruction &I) const {
if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
I.hasMetadata(FALKOR_STRIDED_ACCESS_MD))
return MOStridedAccess;
-
- auto Flags = MachineMemOperand::MONone;
- const MDNode *AtomicStHint = I.getMetadata("aarch64.atomic.hint");
- if (AtomicStHint) {
- unsigned HintVal =
- cast<ConstantInt>(
- cast<ConstantAsMetadata>(AtomicStHint->getOperand(0))->getValue())
- ->getZExtValue();
- AArch64AtomicStoreHint Hint = getAtomicStoreHintFromMD(HintVal);
-
- if (static_cast<unsigned>(Hint) & 0b1)
- Flags |= MOAtomicHintBit0;
- if (static_cast<unsigned>(Hint) & 0b10)
- Flags |= MOAtomicHintBit1;
- }
-
- return Flags;
+ return MachineMemOperand::MONone;
}
bool AArch64TargetLowering::isLegalInterleavedAccessType(
@@ -34085,6 +34069,14 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
return true;
}
}
+
+ // The !mem.cache_hint metadata is not supported by GISel and will be dropped.
+ // TODO: Remove this and handle atomic store hints in GISel once supported.
+ if (auto *Store = dyn_cast<StoreInst>(&Inst)) {
+ if (Store->isAtomic() && getMemCacheHintMetadata(*Store, 1))
+ return true;
+ }
+
return false;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 8ab0c13e41edc..faddeab7868f4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -321,91 +321,93 @@ defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xexten
defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
-class atomic_hint_store<PatFrag Base, int Order, code Pred, code GIPred>
+class atomic_hint_store<PatFrag Base, int Order, code Pred>
: PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
let IsAtomicOrderingMonotonic = !eq(Order, 2);
let IsAtomicOrderingRelease = !eq(Order, 5);
let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
- let GISelPredicateCode = GIPred;
+
+ // TODO: Change once mem.cache_hint supported in GISel
+ let GISelPredicateCode = [{ return false; }];
}
-multiclass AtomicHintPatternsAddrMode<int Hint, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
(ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
(ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
(ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
(ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
(ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
(ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
(ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
(ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
(ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
(ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
(ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
(ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred, GIPred>
+ def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
}
-multiclass AtomicHintPatterns<int Order, int Hint, code Pred, code GIPred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+multiclass AtomicHintPatterns<int Order, int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred, GIPred> GPR32:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred> GPR32:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred, GIPred> GPR64:$data, GPR64sp:$addr),
+ def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred> GPR64:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
}
let AddedComplexity = 15 in {
- defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
-
- defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }], [{ return isAtomicSTSHH_KEEP(MI); }]>;
- defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
- defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }], [{ return isAtomicSTSHH_STRM(MI); }]>;
+ defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
+
+ defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index facff777d9012..921683ca59e11 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -25,6 +25,7 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
+#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
@@ -2997,17 +2998,6 @@ bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
});
}
-AArch64AtomicStoreHint
-AArch64InstrInfo::decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags) {
- unsigned AtomicHint = 0;
- if (MMOFlags & MOAtomicHintBit0)
- AtomicHint += 0b1;
- if (MMOFlags & MOAtomicHintBit1)
- AtomicHint += 0b10;
-
- return static_cast<AArch64AtomicStoreHint>(AtomicHint);
-}
-
bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
switch (Opc) {
default:
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 7bfaf4471b8c0..a7597e69ffb7c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -31,10 +31,6 @@ static const MachineMemOperand::Flags MOSuppressPair =
MachineMemOperand::MOTargetFlag1;
static const MachineMemOperand::Flags MOStridedAccess =
MachineMemOperand::MOTargetFlag2;
-static const MachineMemOperand::Flags MOAtomicHintBit0 =
- MachineMemOperand::MOTargetFlag3;
-static const MachineMemOperand::Flags MOAtomicHintBit1 =
- MachineMemOperand::MOTargetFlag4;
#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
@@ -240,8 +236,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64AtomicStoreHint
- decodeAtomicHintFlags(MachineMemOperand::Flags MMOFlags);
+ static AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand &MMO);
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index c45f1cca2a5a6..8971bee5b2f1d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -530,11 +530,6 @@ class AArch64InstructionSelector : public InstructionSelector {
MachineOperand &Predicate,
MachineIRBuilder &MIRBuilder) const;
- bool isAtomicHintInst(const MachineInstr &MI,
- AArch64AtomicStoreHint Hint) const;
- bool isAtomicSTSHH_KEEP(const MachineInstr &MI) const;
- bool isAtomicSTSHH_STRM(const MachineInstr &MI) const;
-
/// Return true if \p MI is a load or store of \p NumBytes bytes.
bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
@@ -2553,107 +2548,6 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
I.eraseFromParent();
return true;
}
- case TargetOpcode::G_STORE: {
- GStore &St = cast<GStore>(I);
- auto MMO = St.getMMO();
- LLT PtrTy = MRI.getType(St.getPointerReg());
- AArch64AtomicStoreHint Hint = TII.decodeAtomicHintFlags(MMO.getFlags());
-
- // Only for handling atomic store with hint.
- // Can only handle AddressSpace 0, 64-bit pointers.
- if (!St.isAtomic() || PtrTy != LLT::pointer(0, 64) ||
- Hint == AArch64AtomicStoreHint::HINT_NONE) {
- return false;
- }
-
- // AtomicExpandPass converts FP stores to integer stores of the equivalent
- // bitwidth. Widen the register here if the original type was f16.
- unsigned StoreSize = St.getMemSize().getValue();
- Register ValueReg = St.getValueReg();
- if (StoreSize == 2) {
- Register CastReg;
- if (mi_match(ValueReg, MRI, m_GBitcast(m_Reg(CastReg)))) {
- auto Undef = MIB.buildInstr(TargetOpcode::IMPLICIT_DEF,
- {&AArch64::FPR32RegClass}, {});
- auto Ins = MIB.buildInstr(TargetOpcode::INSERT_SUBREG,
- {&AArch64::FPR32RegClass}, {Undef, ValueReg})
- .addImm(AArch64::hsub);
- constrainSelectedInstRegOperands(*Undef, TII, TRI, RBI);
- constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
- ValueReg = Ins.getReg(0);
- }
- }
-
- static constexpr unsigned BaseOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_B, AArch64::ATOMIC_STORE_HINT_H,
- AArch64::ATOMIC_STORE_HINT_S, AArch64::ATOMIC_STORE_HINT_D};
- static constexpr unsigned RegWOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_BroW, AArch64::ATOMIC_STORE_HINT_HroW,
- AArch64::ATOMIC_STORE_HINT_SroW, AArch64::ATOMIC_STORE_HINT_DroW};
- static constexpr unsigned RegXOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_BroX, AArch64::ATOMIC_STORE_HINT_HroX,
- AArch64::ATOMIC_STORE_HINT_SroX, AArch64::ATOMIC_STORE_HINT_DroX};
- static constexpr unsigned UImmOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_Bui, AArch64::ATOMIC_STORE_HINT_Hui,
- AArch64::ATOMIC_STORE_HINT_Sui, AArch64::ATOMIC_STORE_HINT_Dui};
- static constexpr unsigned ImmOpcodes[] = {
- AArch64::ATOMIC_STORE_HINT_Bi, AArch64::ATOMIC_STORE_HINT_Hi,
- AArch64::ATOMIC_STORE_HINT_Si, AArch64::ATOMIC_STORE_HINT_Di};
-
- AtomicOrdering Ordering = MMO.getSuccessOrdering();
- MachineInstrBuilder StrPseudo;
- unsigned HintOpc = 0;
-
- // Other addressing modes can be used when the ordering is monotonic.
- // Try to match these first, before falling back on the basic operands.
- auto AddModeWRO = selectAddrModeWRO(St.getOperand(1), StoreSize);
- auto AddModeXRO = selectAddrModeXRO(St.getOperand(1), StoreSize);
- auto AddModeUImm = selectAddrModeIndexed(St.getOperand(1), StoreSize);
- auto AddModeImm = selectAddrModeUnscaled(St.getOperand(1), StoreSize);
-
- if (AddModeWRO && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = RegWOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeWRO)[0](StrPseudo);
- (*AddModeWRO)[1](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeWRO)[2](StrPseudo);
- } else if (AddModeXRO && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = RegXOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeXRO)[0](StrPseudo);
- (*AddModeXRO)[1](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeXRO)[2](StrPseudo);
- } else if (AddModeUImm && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = UImmOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeUImm)[0](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeUImm)[1](StrPseudo);
- } else if (AddModeImm && Ordering == AtomicOrdering::Monotonic) {
- HintOpc = ImmOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc));
- (*AddModeImm)[0](StrPseudo);
- StrPseudo.addReg(ValueReg);
- (*AddModeImm)[1](StrPseudo);
- } else {
- HintOpc = BaseOpcodes[Log2_32(StoreSize)];
- StrPseudo = BuildMI(MBB, I, MIMetadata(I), TII.get(HintOpc))
- .addReg(St.getPointerReg())
- .addReg(ValueReg);
- }
-
- // Add the ordering and hint operands, before erasing the store.
- unsigned HintImm = Hint == AArch64AtomicStoreHint::HINT_STSHH_KEEP ? 48 : 49;
- StrPseudo.addImm((int)Ordering);
- StrPseudo.addImm(HintImm);
-
- StrPseudo.cloneMemRefs(I);
- I.eraseFromParent();
- constrainSelectedInstRegOperands(*StrPseudo, TII, TRI, RBI);
- return true;
- }
default:
return false;
}
@@ -8442,22 +8336,6 @@ void AArch64InstructionSelector::renderFPImm32SIMDModImmType4(
.getZExtValue()));
}
-bool AArch64InstructionSelector::isAtomicHintInst(
- const MachineInstr &MI, AArch64AtomicStoreHint Hint) const {
- auto MMO = cast<GStore>(MI).getMMO();
- return AArch64InstrInfo::decodeAtomicHintFlags(MMO.getFlags()) == Hint;
-}
-
-bool AArch64InstructionSelector::isAtomicSTSHH_KEEP(
- const MachineInstr &MI) const {
- return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_KEEP);
-}
-
-bool AArch64InstructionSelector::isAtomicSTSHH_STRM(
- const MachineInstr &MI) const {
- return isAtomicHintInst(MI, AArch64AtomicStoreHint::HINT_STSHH_STRM);
-}
-
bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
const MachineInstr &MI, unsigned NumBytes) const {
if (!MI.mayLoadOrStore())
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index 21aec7dae322f..d2c5b388df0b3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; STSHH: Keep, Relaxed
@@ -12,7 +12,7 @@ define void @test_atomic_store_keep_relaxed_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: strb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -22,7 +22,7 @@ define void @test_atomic_store_keep_relaxed_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: strh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -32,7 +32,7 @@ define void @test_atomic_store_keep_relaxed_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: str w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -42,7 +42,7 @@ define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: str x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %ptr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -50,6 +50,18 @@ define void @test_atomic_store_keep_relaxed_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Keep, Release
;
+define void @test_atomic_store_keep_release_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_keep_release_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_keep_release_bfloat:
; CHECK: // %bb.0:
@@ -58,7 +70,7 @@ define void @test_atomic_store_keep_release_bfloat(ptr %ptr, bfloat %val) nounwi
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -70,7 +82,7 @@ define void @test_atomic_store_keep_release_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -81,7 +93,7 @@ define void @test_atomic_store_keep_release_float(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -92,7 +104,7 @@ define void @test_atomic_store_keep_release_double(ptr %ptr, double %val) nounwi
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr release, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %ptr release, align 8, !mem.cache_hint !0
ret void
}
@@ -106,7 +118,7 @@ define void @test_atomic_store_keep_seqcst_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -116,7 +128,7 @@ define void @test_atomic_store_keep_seqcst_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlrh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -126,7 +138,7 @@ define void @test_atomic_store_keep_seqcst_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -136,7 +148,7 @@ define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh keep
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !0
ret void
}
@@ -144,6 +156,18 @@ define void @test_atomic_store_keep_seqcst_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Stream, Relaxed
;
+define void @test_atomic_store_strm_relaxed_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_strm_relaxed_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
; CHECK: // %bb.0:
@@ -152,7 +176,7 @@ define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwi
; CHECK-NEXT: stshh strm
; CHECK-NEXT: strh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -164,7 +188,7 @@ define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: strh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -175,7 +199,7 @@ define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh strm
; CHECK-NEXT: str w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -186,7 +210,7 @@ define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwi
; CHECK-NEXT: stshh strm
; CHECK-NEXT: str x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -200,7 +224,7 @@ define void @test_atomic_store_stream_release_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -210,7 +234,7 @@ define void @test_atomic_store_stream_release_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w1, [x0]
; CHECK-NEXT: ret
- store atomic i16 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -220,7 +244,7 @@ define void @test_atomic_store_stream_release_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr w1, [x0]
; CHECK-NEXT: ret
- store atomic i32 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -230,7 +254,7 @@ define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr x1, [x0]
; CHECK-NEXT: ret
- store atomic i64 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %ptr release, align 8, !mem.cache_hint !2
ret void
}
@@ -238,6 +262,18 @@ define void @test_atomic_store_stream_release_i64(ptr %ptr, i64 %val) nounwind {
; STSHH: Stream, SequentiallyConsistent
;
+define void @test_atomic_store_stream_seqcst_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: test_atomic_store_stream_seqcst_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: stlrb w8, [x0]
+; CHECK-NEXT: ret
+ store atomic <1 x i8> %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_stream_seqcst_bfloat:
; CHECK: // %bb.0:
@@ -246,7 +282,7 @@ define void @test_atomic_store_stream_seqcst_bfloat(ptr %ptr, bfloat %val) nounw
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic bfloat %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -258,7 +294,7 @@ define void @test_atomic_store_stream_seqcst_i16(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlrh w8, [x0]
; CHECK-NEXT: ret
- store atomic half %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -269,7 +305,7 @@ define void @test_atomic_store_stream_seqcst_i32(ptr %ptr, float %val) nounwind
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr w8, [x0]
; CHECK-NEXT: ret
- store atomic float %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -280,7 +316,7 @@ define void @test_atomic_store_stream_seqcst_double(ptr %ptr, double %val) nounw
; CHECK-NEXT: stshh strm
; CHECK-NEXT: stlr x8, [x0]
; CHECK-NEXT: ret
- store atomic double %val, ptr %ptr seq_cst, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
ret void
}
@@ -293,10 +329,13 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: stlrb w1, [x0]
; CHECK-NEXT: ret
- store atomic i8 %val, ptr %ptr release, align 8, !aarch64.atomic.hint !2
+ store atomic i8 %val, ptr %ptr release, align 8, !mem.cache_hint !4
ret void
}
-!0 = !{i32 0}
-!1 = !{i32 1}
-!2 = !{i32 2}
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.atomic_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.atomic_hint", i32 1}
+!4 = !{i32 1, !5}
+!5 = !{!"aarch64.atomic_hint", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index d80d9460c107c..a6dea2b9f511e 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -global-isel=1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel=1 -global-isel-abort=0 -verify-machineinstrs < %s | FileCheck %s
;
; W register offset
@@ -15,7 +15,7 @@ define void @relaxed_store_hint_roW_i8(ptr %ptr, i32 %offset, i8 %val) nounwind
; CHECK-NEXT: strb w2, [x0, w1, sxtw]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -26,7 +26,7 @@ define void @relaxed_store_hint_roW_i16(ptr %ptr, i32 %offset, i16 %val) nounwin
; CHECK-NEXT: strh w2, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 %offset
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -37,7 +37,7 @@ define void @relaxed_store_hint_roW_i32(ptr %ptr, i32 %offset, i32 %val) nounwin
; CHECK-NEXT: str w2, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 %offset
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -48,12 +48,25 @@ define void @relaxed_store_hint_roW_i64(ptr %ptr, i32 %offset, i64 %val) nounwin
; CHECK-NEXT: str x2, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 %offset
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
; FP
+define void @relaxed_store_hint_roW_mfloat(ptr %ptr, i32 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roW_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: strb w8, [x0, w1, sxtw]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
; CHECK: // %bb.0:
@@ -63,7 +76,7 @@ define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) n
; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 %offset
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -76,7 +89,7 @@ define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounw
; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 %offset
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -88,7 +101,7 @@ define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nou
; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 %offset
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -100,7 +113,7 @@ define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) n
; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 %offset
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -117,7 +130,7 @@ define void @relaxed_store_hint_roX_i8(ptr %ptr, i64 %offset, i8 %val) nounwind
; CHECK-NEXT: strb w2, [x0, x1]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i64 %offset
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -128,7 +141,7 @@ define void @relaxed_store_hint_roX_i16(ptr %ptr, i64 %offset, i16 %val) nounwin
; CHECK-NEXT: strh w2, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i64 %offset
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -139,7 +152,7 @@ define void @relaxed_store_hint_roX_i32(ptr %ptr, i64 %offset, i32 %val) nounwin
; CHECK-NEXT: str w2, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i64 %offset
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -150,12 +163,25 @@ define void @relaxed_store_hint_roX_i64(ptr %ptr, i64 %offset, i64 %val) nounwin
; CHECK-NEXT: str x2, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i64 %offset
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
; FP
+define void @relaxed_store_hint_roX_mfloat(ptr %ptr, i64 %offset, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_roX_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, x1]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i64 %offset
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
; CHECK: // %bb.0:
@@ -165,7 +191,7 @@ define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) n
; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i64 %offset
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -178,7 +204,7 @@ define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounw
; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i64 %offset
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -190,7 +216,7 @@ define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nou
; CHECK-NEXT: str w8, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i64 %offset
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -202,7 +228,7 @@ define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) n
; CHECK-NEXT: str x8, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i64 %offset
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -219,7 +245,7 @@ define void @relaxed_store_hint_uimm_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: strb w1, [x0, #4095]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 4095
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -230,7 +256,7 @@ define void @relaxed_store_hint_uimm_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: strh w1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 2048
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -241,7 +267,7 @@ define void @relaxed_store_hint_uimm_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: str w1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 1024
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -252,12 +278,25 @@ define void @relaxed_store_hint_uimm_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: str x1, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 512
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
; FP
+define void @relaxed_store_hint_uimm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_uimm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh keep
+; CHECK-NEXT: strb w8, [x0, #2048]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 2048
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
+ ret void
+}
+
define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
; CHECK: // %bb.0:
@@ -267,7 +306,7 @@ define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-NEXT: strh w8, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 2048
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -280,7 +319,7 @@ define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: strh w8, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 2048
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -292,7 +331,7 @@ define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
; CHECK-NEXT: str w8, [x0, #8192]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 2048
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -304,7 +343,7 @@ define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
; CHECK-NEXT: str x8, [x0, #16384]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 2048
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -321,7 +360,7 @@ define void @relaxed_store_hint_imm_i8(ptr %ptr, i8 %val) nounwind {
; CHECK-NEXT: sturb w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i8, ptr %ptr, i32 -256
- store atomic i8 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i8 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -332,7 +371,7 @@ define void @relaxed_store_hint_imm_i16(ptr %ptr, i16 %val) nounwind {
; CHECK-NEXT: sturh w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i16, ptr %ptr, i32 -128
- store atomic i16 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i16 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -343,7 +382,7 @@ define void @relaxed_store_hint_imm_i32(ptr %ptr, i32 %val) nounwind {
; CHECK-NEXT: stur w1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i32, ptr %ptr, i32 -64
- store atomic i32 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i32 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
@@ -354,12 +393,25 @@ define void @relaxed_store_hint_imm_i64(ptr %ptr, i64 %val) nounwind {
; CHECK-NEXT: stur x1, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr i64, ptr %ptr, i32 -32
- store atomic i64 %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !0
+ store atomic i64 %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
ret void
}
; FP
+define void @relaxed_store_hint_imm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
+; CHECK-LABEL: relaxed_store_hint_imm_mfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: umov w8, v0.b[0]
+; CHECK-NEXT: stshh strm
+; CHECK-NEXT: sturb w8, [x0, #-128]
+; CHECK-NEXT: ret
+ %addr = getelementptr <1 x i8>, ptr %ptr, i32 -128
+ store atomic <1 x i8> %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
+ ret void
+}
+
define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
; CHECK: // %bb.0:
@@ -369,7 +421,7 @@ define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-NEXT: sturh w8, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 -128
- store atomic bfloat %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -382,7 +434,7 @@ define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
; CHECK-NEXT: sturh w8, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 -128
- store atomic half %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -395,7 +447,7 @@ define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
; CHECK-NEXT: str w9, [x8]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 -128
- store atomic float %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
@@ -408,9 +460,11 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
; CHECK-NEXT: str x9, [x8]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 -128
- store atomic double %val, ptr %addr monotonic, align 8, !aarch64.atomic.hint !1
+ store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
ret void
}
-!0 = !{ i32 0 }
-!1 = !{ i32 1 }
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.atomic_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.atomic_hint", i32 1}
>From e47d181bcee4c8f8e71164913aff16cf2c279656 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Tue, 25 Aug 2026 14:53:01 +0000
Subject: [PATCH 07/13] - Reduce number of patterns by replacing Order with
Relaxed - Add floating-point patterns for relaxed stores - Renamed
aarch64.atomic_hint -> aarch64.mem_hint - Added warning for invalid types
passed to builtin for hint argument - Rename getAtomicStoreHintFromMD ->
toAArch64Hint - Cleaned up includes
---
.../clang/Basic/DiagnosticSemaKinds.td | 2 +
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 10 +-
clang/lib/Sema/SemaARM.cpp | 13 +-
clang/test/CodeGen/arm_acle.c | 4 +-
clang/test/CodeGen/builtins-arm64.c | 4 +-
clang/test/Sema/builtins-arm64.c | 4 +-
.../include/llvm/Support/AArch64AtomicHints.h | 36 -----
.../include/llvm/Support/AArch64MemoryHints.h | 35 +++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 66 +++++---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 36 ++---
.../lib/Target/AArch64/AArch64InstrAtomics.td | 148 ++++++++++--------
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 1 -
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 4 +-
.../GISel/AArch64InstructionSelector.cpp | 1 -
.../Atomics/aarch64-atomic-store-hint.ll | 12 +-
.../Atomics/aarch64-relaxed-store-hint.ll | 28 ++--
16 files changed, 221 insertions(+), 183 deletions(-)
delete mode 100644 llvm/include/llvm/Support/AArch64AtomicHints.h
create mode 100644 llvm/include/llvm/Support/AArch64MemoryHints.h
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9baa201c05239..e854652a9efb8 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9619,6 +9619,8 @@ def err_atomic_builtin_must_be_pointer_intptr : Error<
def err_atomic_hint_builtin_must_be_pointer : Error<
"address argument to atomic hint builtin must be a pointer to a scalar "
"integral or floating-point type of 8, 16, 32, or 64 bits (%0 invalid)">;
+def err_atomic_hint_has_invalid_hint_type : Error<
+ "invalid hint type argument to atomic hint operation (%0)">;
def err_atomic_builtin_cannot_be_const : Error<
"address argument to atomic builtin cannot be const-qualified (%0 invalid)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 8a4f19e4e016d..18d7ea3d2f6de 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -20,7 +20,7 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsBPF.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/TargetParser/AArch64TargetParser.h"
#include <numeric>
@@ -2105,14 +2105,14 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned HintArg = Result.Val.getInt().getExtValue();
// Attach the hint if valid
- if (getAtomicStoreHintFromMD(HintArg) != AArch64AtomicStoreHint::HINT_NONE) {
+ if (toAArch64Hint(HintArg) != AArch64MemoryHint::HINT_NONE) {
LLVMContext &Ctx = CGM.getLLVMContext();
- MDNode *AtomicHint = MDNode::get(
- Ctx, {MDString::get(Ctx, "aarch64.atomic_hint"),
+ MDNode *MemHint = MDNode::get(
+ Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
llvm::ConstantAsMetadata::get(Builder.getInt32(HintArg))});
MDNode *HintNode = MDNode::get(
CGM.getLLVMContext(),
- {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), AtomicHint});
+ {llvm::ConstantAsMetadata::get(Builder.getInt32(1)), MemHint});
Store->setMetadata(llvm::LLVMContext::MD_mem_cache_hint, HintNode);
}
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index b3c139bf1c8bc..e68f4868e7698 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -17,7 +17,7 @@
#include "clang/Sema/Initialization.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
namespace clang {
@@ -343,7 +343,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
QualType PtrQT = PtrTy->getPointeeType();
- if (!PtrQT->isIntegralType(getASTContext()) && !PtrQT->isFloatingType() &&
+ if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
!PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_op_needs_atomic_int_or_fp)
@@ -391,20 +391,19 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_hint_has_invalid_memory_order)
<< *MemOrdAP << MemOrdArg->getSourceRange();
- // Arg 3 is the hint type. Only values represented by AArch64AtomicStoreHint
+ // Arg 3 is the hint type. Only values represented by AArch64MemoryHint
// are valid.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP) {
- Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ if (!HintAP || !HintArg->getType()->isIntegralType(Context)) {
+ Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
return false;
}
unsigned Hint = HintAP->getZExtValue();
- if (llvm::getAtomicStoreHintFromMD(Hint) ==
- llvm::AArch64AtomicStoreHint::HINT_NONE)
+ if (llvm::toAArch64Hint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 041530a5d2269..e8e9d4ee72534 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1936,8 +1936,8 @@ void test_atomic_store_hint_double(double *ptr, double data) {
}
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
-// AArch64-NEXT: [[HINT2]] = !{!"aarch64.atomic_hint", i32 0}
+// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
// AArch64-NEXT: [[HINT3]] = !{i32 1, [[HINT4:![0-9]+]]}
-// AArch64-NEXT: [[HINT4]] = !{!"aarch64.atomic_hint", i32 1}
+// AArch64-NEXT: [[HINT4]] = !{!"aarch64.mem_hint", i32 1}
#endif
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 6e11bc652ae9d..672dbe016a1cd 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -233,6 +233,6 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
// CHECK: ![[M0]] = !{!"1:2:3:4:5"}
// CHECK: ![[M1]] = !{i32 1, ![[M2:[0-9]+]]}
-// CHECK: ![[M2]] = !{!"aarch64.atomic_hint", i32 0}
+// CHECK: ![[M2]] = !{!"aarch64.mem_hint", i32 0}
// CHECK: ![[M3]] = !{i32 1, ![[M4:[0-9]+]]}
-// CHECK: ![[M4]] = !{!"aarch64.atomic_hint", i32 1}
+// CHECK: ![[M4]] = !{!"aarch64.mem_hint", i32 1}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 5372aff24b739..c665507220a67 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -65,6 +65,8 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-warning {{unrecognised hint type argument to atomic hint operation ('int')}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{invalid hint type argument to atomic hint operation ('char *')}}
}
diff --git a/llvm/include/llvm/Support/AArch64AtomicHints.h b/llvm/include/llvm/Support/AArch64AtomicHints.h
deleted file mode 100644
index 8118f3e2df3ad..0000000000000
--- a/llvm/include/llvm/Support/AArch64AtomicHints.h
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- AArch64AtomicHints.h - AArch64 Atomic Hint Attributes ---*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_AARCH64ATOMICHINTS_H
-#define LLVM_SUPPORT_AARCH64ATOMICHINTS_H
-
-namespace llvm {
-enum class AArch64AtomicStoreHint {
- HINT_NONE = 0,
- HINT_STSHH_KEEP = 1,
- HINT_STSHH_STRM = 2,
-};
-
-template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
- return (Int)AArch64AtomicStoreHint::HINT_STSHH_KEEP <= I &&
- I <= (Int)AArch64AtomicStoreHint::HINT_STSHH_STRM;
-}
-
-template <typename Int>
-inline AArch64AtomicStoreHint getAtomicStoreHintFromMD(Int I) {
- switch (I) {
- case 0:
- return AArch64AtomicStoreHint::HINT_STSHH_KEEP;
- case 1:
- return AArch64AtomicStoreHint::HINT_STSHH_STRM;
- default:
- return AArch64AtomicStoreHint::HINT_NONE;
- }
-}
-} // namespace llvm
-#endif // LLVM_SUPPORT_AARCH64ATOMICHINTS_H
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
new file mode 100644
index 0000000000000..9efc68faf0718
--- /dev/null
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -0,0 +1,35 @@
+//===-- AArch64MemoryHints.h - AArch64 Memory Hint Attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+#define LLVM_SUPPORT_AARCH64MEMORYHINTS_H
+
+namespace llvm {
+enum class AArch64MemoryHint {
+ HINT_NONE = 0,
+ HINT_STSHH_KEEP = 1,
+ HINT_STSHH_STRM = 2,
+};
+
+template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+ return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I &&
+ I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM;
+}
+
+template <typename Int> inline AArch64MemoryHint toAArch64Hint(Int I) {
+ switch (I) {
+ case 0:
+ return AArch64MemoryHint::HINT_STSHH_KEEP;
+ case 1:
+ return AArch64MemoryHint::HINT_STSHH_STRM;
+ default:
+ return AArch64MemoryHint::HINT_NONE;
+ }
+}
+} // namespace llvm
+#endif // LLVM_SUPPORT_AARCH64MEMORYHINTS_H
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 426805cb5d42f..4d1932183cacb 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3257,8 +3257,8 @@ void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(2).getImm();
- bool Relaxed = Order == 2;
+ unsigned Relaxed = MI->getOperand(2).getImm();
+ assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
switch (MI->getOpcode()) {
case AArch64::ATOMIC_STORE_HINT_B:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
@@ -3266,10 +3266,10 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_H:
StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
break;
- case AArch64::ATOMIC_STORE_HINT_S:
+ case AArch64::ATOMIC_STORE_HINT_W:
StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
break;
- case AArch64::ATOMIC_STORE_HINT_D:
+ case AArch64::ATOMIC_STORE_HINT_X:
StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
break;
default:
@@ -3292,8 +3292,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(5).getImm();
- assert(Order == 2 &&
+ assert(MI->getOperand(5).getImm() == 1 &&
"Atomic store addressing mode only supports relaxed stores");
switch (MI->getOpcode()) {
@@ -3303,24 +3302,36 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
case AArch64::ATOMIC_STORE_HINT_HroW:
StOpc = AArch64::STRHHroW;
break;
- case AArch64::ATOMIC_STORE_HINT_SroW:
+ case AArch64::ATOMIC_STORE_HINT_WroW:
StOpc = AArch64::STRWroW;
break;
- case AArch64::ATOMIC_STORE_HINT_DroW:
+ case AArch64::ATOMIC_STORE_HINT_XroW:
StOpc = AArch64::STRXroW;
break;
+ case AArch64::ATOMIC_STORE_HINT_SroW:
+ StOpc = AArch64::STRSroW;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroW:
+ StOpc = AArch64::STRDroW;
+ break;
case AArch64::ATOMIC_STORE_HINT_BroX:
StOpc = AArch64::STRBBroX;
break;
case AArch64::ATOMIC_STORE_HINT_HroX:
StOpc = AArch64::STRHHroX;
break;
- case AArch64::ATOMIC_STORE_HINT_SroX:
+ case AArch64::ATOMIC_STORE_HINT_WroX:
StOpc = AArch64::STRWroX;
break;
- case AArch64::ATOMIC_STORE_HINT_DroX:
+ case AArch64::ATOMIC_STORE_HINT_XroX:
StOpc = AArch64::STRXroX;
break;
+ case AArch64::ATOMIC_STORE_HINT_SroX:
+ StOpc = AArch64::STRSroX;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_DroX:
+ StOpc = AArch64::STRDroX;
+ break;
default:
llvm_unreachable("Unexpected atomic hint opcode.");
}
@@ -3342,8 +3353,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
const MachineInstr *MI) {
unsigned StOpc;
- unsigned Order = MI->getOperand(3).getImm();
- assert(Order == 2 &&
+ assert(MI->getOperand(3).getImm() == 1 &&
"Atomic store addressing mode only supports relaxed stores");
switch (MI->getOpcode()) {
@@ -3353,24 +3363,36 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
case AArch64::ATOMIC_STORE_HINT_Hui:
StOpc = AArch64::STRHHui;
break;
- case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
StOpc = AArch64::STRWui;
break;
- case AArch64::ATOMIC_STORE_HINT_Dui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
StOpc = AArch64::STRXui;
break;
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ StOpc = AArch64::STRSui;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Dui:
+ StOpc = AArch64::STRDui;
+ break;
case AArch64::ATOMIC_STORE_HINT_Bi:
StOpc = AArch64::STURBBi;
break;
case AArch64::ATOMIC_STORE_HINT_Hi:
StOpc = AArch64::STURHHi;
break;
- case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
StOpc = AArch64::STURWi;
break;
- case AArch64::ATOMIC_STORE_HINT_Di:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
StOpc = AArch64::STURXi;
break;
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ StOpc = AArch64::STURSi;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ StOpc = AArch64::STURDi;
+ break;
default:
llvm_unreachable("Unexpected atomic hint opcode.");
}
@@ -4106,26 +4128,34 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
return;
case AArch64::ATOMIC_STORE_HINT_B:
case AArch64::ATOMIC_STORE_HINT_H:
- case AArch64::ATOMIC_STORE_HINT_S:
- case AArch64::ATOMIC_STORE_HINT_D:
+ case AArch64::ATOMIC_STORE_HINT_W:
+ case AArch64::ATOMIC_STORE_HINT_X:
emitAtomicHintPseudoExpansion(MI);
return;
case AArch64::ATOMIC_STORE_HINT_BroW:
case AArch64::ATOMIC_STORE_HINT_HroW:
+ case AArch64::ATOMIC_STORE_HINT_WroW:
+ case AArch64::ATOMIC_STORE_HINT_XroW:
case AArch64::ATOMIC_STORE_HINT_SroW:
case AArch64::ATOMIC_STORE_HINT_DroW:
case AArch64::ATOMIC_STORE_HINT_BroX:
case AArch64::ATOMIC_STORE_HINT_HroX:
+ case AArch64::ATOMIC_STORE_HINT_WroX:
+ case AArch64::ATOMIC_STORE_HINT_XroX:
case AArch64::ATOMIC_STORE_HINT_SroX:
case AArch64::ATOMIC_STORE_HINT_DroX:
emitAtomicHintPseudoExpansionRO(MI);
return;
case AArch64::ATOMIC_STORE_HINT_Bui:
case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
case AArch64::ATOMIC_STORE_HINT_Sui:
case AArch64::ATOMIC_STORE_HINT_Dui:
case AArch64::ATOMIC_STORE_HINT_Bi:
case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
case AArch64::ATOMIC_STORE_HINT_Si:
case AArch64::ATOMIC_STORE_HINT_Di:
emitAtomicHintPseudoExpansionImm(MI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index b942b4756c704..3a306ca17a6f5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -24,7 +24,7 @@
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
@@ -518,7 +518,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
bool SelectCMP_SWAP(SDNode *N);
- AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand *MMO) const;
+ AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
bool isAtomicSTSHH_KEEP(SDNode *N) const;
bool isAtomicSTSHH_STRM(SDNode *N) const;
@@ -4620,36 +4620,32 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
return true;
}
-AArch64AtomicStoreHint
-AArch64DAGToDAGISel::decodeAtomicHintFlags(MachineMemOperand *MMO) const {
- int AtomicHint = -1;
+AArch64MemoryHint
+AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
+ int MemoryHint = -1;
const MDNode *MemCacheHint = MMO->getMemCacheHint();
if (!MemCacheHint)
- return AArch64AtomicStoreHint::HINT_NONE;
+ return AArch64MemoryHint::HINT_NONE;
for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
- const auto *Key = cast<MDString>(MemCacheHint->getOperand(I));
- StringRef KeyStr = Key->getString();
- const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
-
- if (KeyStr != "aarch64.atomic_hint")
- continue;
-
- AtomicHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
- ->getZExtValue();
+ if (MemCacheHint->getOperand(I).equalsStr("aarch64.mem_hint")) {
+ const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
+ MemoryHint = cast<ConstantInt>(cast<ConstantAsMetadata>(Val)->getValue())
+ ->getZExtValue();
+ }
}
- return getAtomicStoreHintFromMD(AtomicHint);
+ return toAArch64Hint(MemoryHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
- return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64AtomicStoreHint::HINT_STSHH_KEEP;
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_KEEP;
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_STRM(SDNode *N) const {
- return decodeAtomicHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
- AArch64AtomicStoreHint::HINT_STSHH_STRM;
+ return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) ==
+ AArch64MemoryHint::HINT_STSHH_STRM;
}
bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index faddeab7868f4..e0e65c400a8b0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -289,23 +289,23 @@ def : Pat<(relaxed_store<atomic_store_64>
let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
class BaseStoreHintPseudo<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
: Pseudo<(outs), (ins GPR64sp:$Rn, GPR32:$Rm, regtype:$data, WExt:$extend,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
: Pseudo<(outs), (ins GPR64sp:$Rn, GPR64:$Rm, regtype:$data, XExt:$extend,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
: Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, UImmOp:$offset,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoImm<RegisterClass regtype>
: Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, simm9:$offset,
- i32imm:$order, i32imm:$hint), []>, Sched<[WriteAtomic]>;
+ i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
}
multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand XExt, Operand UImmOp> {
@@ -318,96 +318,118 @@ multiclass AtomicStoreHintPseudos<RegisterClass regtype, Operand WExt, Operand X
defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xextend8, uimm12s1>;
defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
-defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
-defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_W : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_X : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<FPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
+defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<FPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
-class atomic_hint_store<PatFrag Base, int Order, code Pred>
+class atomic_hint_store<PatFrag Base, int Relaxed, code Pred>
: PatFrag<(ops node:$val, node:$ptr),
(Base node:$val, node:$ptr), Pred> {
let IsAtomic = 1;
- let IsAtomicOrderingMonotonic = !eq(Order, 2);
- let IsAtomicOrderingRelease = !eq(Order, 5);
- let IsAtomicOrderingSequentiallyConsistent = !eq(Order, 7);
+ let IsAtomicOrderingReleaseOrStronger = !eq(Relaxed, 0);
// TODO: Change once mem.cache_hint supported in GISel
let GISelPredicateCode = [{ return false; }];
}
multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
- (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
- (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
- (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_WroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
- (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_XroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, FPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, FPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
- (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
- (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
- (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_WroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
- (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_XroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, FPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, FPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
- (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
- (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
- (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_Wui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
- (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 2), (i32 Hint))>;
-
- def : Pat<(atomic_hint_store<atomic_store_8, 2, Pred>
+ (ATOMIC_STORE_HINT_Xui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, FPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, FPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+
+ def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, 2, Pred>
+ (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, 2, Pred>
+ (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Si GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 2), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, 2, Pred>
+ (ATOMIC_STORE_HINT_Wi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Di GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 2), (i32 Hint))>;
-}
-
-multiclass AtomicHintPatterns<int Order, int Hint, code Pred> {
- def : Pat<(atomic_hint_store<atomic_store_8, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_16, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_32, Order, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_S GPR64sp:$addr, GPR32:$data, (i32 Order), (i32 Hint))>;
- def : Pat<(atomic_hint_store<atomic_store_64, Order, Pred> GPR64:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_D GPR64sp:$addr, GPR64:$data, (i32 Order), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Xi GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
+ (i32 (bitconvert (f32 FPR32Op:$data))), (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si GPR64sp:$addr, FPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
+ (i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di GPR64sp:$addr, FPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+}
+
+multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
+ def : Pat<(atomic_hint_store<atomic_store_8, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_16, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_32, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_W GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_X GPR64sp:$addr, GPR64:$data, (i32 Relaxed), (i32 Hint))>;
}
let AddedComplexity = 15 in {
defm : AtomicHintPatternsAddrMode<48, [{ return isAtomicSTSHH_KEEP(N); }]>;
defm : AtomicHintPatternsAddrMode<49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<2, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<5, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<7, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
- defm : AtomicHintPatterns<2, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<5, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
- defm : AtomicHintPatterns<7, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<1, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<0, 48, [{ return isAtomicSTSHH_KEEP(N); }]>;
+ defm : AtomicHintPatterns<1, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
+ defm : AtomicHintPatterns<0, 49, [{ return isAtomicSTSHH_STRM(N); }]>;
}
//===----------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 921683ca59e11..b8e74f8ad496d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -25,7 +25,6 @@
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
-#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index a7597e69ffb7c..f53dc389ed208 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -16,7 +16,7 @@
#include "AArch64.h"
#include "AArch64RegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/Support/AArch64AtomicHints.h"
+#include "llvm/Support/AArch64MemoryHints.h"
#include "llvm/Support/TypeSize.h"
#include <optional>
@@ -236,7 +236,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64AtomicStoreHint decodeAtomicHintFlags(MachineMemOperand &MMO);
+ static AArch64MemoryHint decodeMemoryFlags(MachineMemOperand &MMO);
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 8971bee5b2f1d..f42e1864d41b5 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -45,7 +45,6 @@
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Type.h"
#include "llvm/Pass.h"
-#include "llvm/Support/AArch64AtomicHints.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index d2c5b388df0b3..bfa9fc49cf752 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -195,9 +195,8 @@ define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w8, [x0]
+; CHECK-NEXT: str s0, [x0]
; CHECK-NEXT: ret
store atomic float %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
@@ -206,9 +205,8 @@ define void @test_atomic_store_strm_relaxed_float(ptr %ptr, float %val) nounwind
define void @test_atomic_store_strm_relaxed_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x8, [x0]
+; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret
store atomic double %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
@@ -334,8 +332,8 @@ define void @test_atomic_store_invalid_hint(ptr %ptr, i8 %val) nounwind {
}
!0 = !{i32 1, !1}
-!1 = !{!"aarch64.atomic_hint", i32 0}
+!1 = !{!"aarch64.mem_hint", i32 0}
!2 = !{i32 1, !3}
-!3 = !{!"aarch64.atomic_hint", i32 1}
+!3 = !{!"aarch64.mem_hint", i32 1}
!4 = !{i32 1, !5}
-!5 = !{!"aarch64.atomic_hint", i32 2}
+!5 = !{!"aarch64.mem_hint", i32 2}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index a6dea2b9f511e..5f5bc43039778 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -96,9 +96,8 @@ define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounw
define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w8, [x0, w1, sxtw #2]
+; CHECK-NEXT: str s0, [x0, w1, sxtw #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 %offset
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -108,9 +107,8 @@ define void @relaxed_store_hint_roW_float(ptr %ptr, i32 %offset, float %val) nou
define void @relaxed_store_hint_roW_double(ptr %ptr, i32 %offset, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x8, [x0, w1, sxtw #3]
+; CHECK-NEXT: str d0, [x0, w1, sxtw #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 %offset
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -211,9 +209,8 @@ define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounw
define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str w8, [x0, x1, lsl #2]
+; CHECK-NEXT: str s0, [x0, x1, lsl #2]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i64 %offset
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -223,9 +220,8 @@ define void @relaxed_store_hint_roX_float(ptr %ptr, i64 %offset, float %val) nou
define void @relaxed_store_hint_roX_double(ptr %ptr, i64 %offset, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str x8, [x0, x1, lsl #3]
+; CHECK-NEXT: str d0, [x0, x1, lsl #3]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i64 %offset
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -326,9 +322,8 @@ define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str w8, [x0, #8192]
+; CHECK-NEXT: str s0, [x0, #8192]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 2048
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -338,9 +333,8 @@ define void @relaxed_store_hint_uimm_float(ptr %ptr, float %val) nounwind {
define void @relaxed_store_hint_uimm_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x8, d0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: str x8, [x0, #16384]
+; CHECK-NEXT: str d0, [x0, #16384]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 2048
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -441,10 +435,9 @@ define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_float:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov w9, s0
; CHECK-NEXT: sub x8, x0, #512
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str w9, [x8]
+; CHECK-NEXT: str s0, [x8]
; CHECK-NEXT: ret
%addr = getelementptr float, ptr %ptr, i32 -128
store atomic float %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -454,10 +447,9 @@ define void @relaxed_store_hint_imm_float(ptr %ptr, float %val) nounwind {
define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: fmov x9, d0
; CHECK-NEXT: sub x8, x0, #1024
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: str x9, [x8]
+; CHECK-NEXT: str d0, [x8]
; CHECK-NEXT: ret
%addr = getelementptr double, ptr %ptr, i32 -128
store atomic double %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -465,6 +457,6 @@ define void @relaxed_store_hint_imm_double(ptr %ptr, double %val) nounwind {
}
!0 = !{i32 1, !1}
-!1 = !{!"aarch64.atomic_hint", i32 0}
+!1 = !{!"aarch64.mem_hint", i32 0}
!2 = !{i32 1, !3}
-!3 = !{!"aarch64.atomic_hint", i32 1}
+!3 = !{!"aarch64.mem_hint", i32 1}
>From 17404060044a108d2b4d8f0a7c6cc530d3029ce3 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 2 Sep 2026 10:57:36 +0000
Subject: [PATCH 08/13] - Remove unnecessary headers & decodeMemoryFlags
declaration
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 2 +-
clang/lib/Sema/SemaARM.cpp | 4 ++--
llvm/include/llvm/Support/AArch64MemoryHints.h | 4 ++--
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 3 +--
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 --
5 files changed, 6 insertions(+), 9 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 18d7ea3d2f6de..4b3c7d2e409b7 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2105,7 +2105,7 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
unsigned HintArg = Result.Val.getInt().getExtValue();
// Attach the hint if valid
- if (toAArch64Hint(HintArg) != AArch64MemoryHint::HINT_NONE) {
+ if (toAArch64MemoryHint(HintArg) != AArch64MemoryHint::HINT_NONE) {
LLVMContext &Ctx = CGM.getLLVMContext();
MDNode *MemHint = MDNode::get(
Ctx, {MDString::get(Ctx, "aarch64.mem_hint"),
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index e68f4868e7698..1159b527c9a9e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -396,14 +396,14 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP || !HintArg->getType()->isIntegralType(Context)) {
+ if (!HintAP) {
Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
<< HintArg->getType() << HintArg->getSourceRange();
return false;
}
unsigned Hint = HintAP->getZExtValue();
- if (llvm::toAArch64Hint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
+ if (llvm::toAArch64MemoryHint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
<< *HintAP << HintArg->getSourceRange();
diff --git a/llvm/include/llvm/Support/AArch64MemoryHints.h b/llvm/include/llvm/Support/AArch64MemoryHints.h
index 9efc68faf0718..44ac64e481e80 100644
--- a/llvm/include/llvm/Support/AArch64MemoryHints.h
+++ b/llvm/include/llvm/Support/AArch64MemoryHints.h
@@ -16,12 +16,12 @@ enum class AArch64MemoryHint {
HINT_STSHH_STRM = 2,
};
-template <typename Int> inline bool isValidAArch64AtomicHintValue(Int I) {
+template <typename Int> inline bool isValidAArch64MemoryHintValue(Int I) {
return (Int)AArch64MemoryHint::HINT_STSHH_KEEP <= I &&
I <= (Int)AArch64MemoryHint::HINT_STSHH_STRM;
}
-template <typename Int> inline AArch64MemoryHint toAArch64Hint(Int I) {
+template <typename Int> inline AArch64MemoryHint toAArch64MemoryHint(Int I) {
switch (I) {
case 0:
return AArch64MemoryHint::HINT_STSHH_KEEP;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 3a306ca17a6f5..ee87070c12727 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -12,7 +12,6 @@
#include "AArch64.h"
#include "AArch64ExpandImm.h"
-#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
@@ -4635,7 +4634,7 @@ AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
}
}
- return toAArch64Hint(MemoryHint);
+ return toAArch64MemoryHint(MemoryHint);
}
bool AArch64DAGToDAGISel::isAtomicSTSHH_KEEP(SDNode *N) const {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index f53dc389ed208..6a8346395eea1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -236,8 +236,6 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
/// Return true if the given load or store is a strided memory access.
static bool isStridedAccess(const MachineInstr &MI);
- static AArch64MemoryHint decodeMemoryFlags(MachineMemOperand &MMO);
-
/// Return true if it has an unscaled load/store offset.
static bool hasUnscaledLdStOffset(unsigned Opc);
static bool hasUnscaledLdStOffset(MachineInstr &MI) {
>From ecc2a8aaa3d6d2c91efa3f6bf883fec541fc8aba Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Fri, 4 Sep 2026 14:11:49 +0000
Subject: [PATCH 09/13] - Fix stack-addressed hinted stores & add tests - Fix
SemaARM to compare unqualified types - Ensure volatile is not dropped when
creating the atomic store - Handle templated arguments to the builtin - Fix
SemaARM for array to pointer conversion - Support bool types - Prevent LFI
seperating stshh from store instruction
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 13 ++-
clang/lib/Sema/SemaARM.cpp | 82 +++++++++++--------
.../AArch64/atomic-store-hint-template.cpp | 38 +++++++++
clang/test/CodeGen/arm_acle.c | 54 ++++++++++++
clang/test/Sema/builtins-arm64.c | 7 +-
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 6 +-
.../lib/Target/AArch64/AArch64InstrAtomics.td | 66 +++++++--------
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 24 ++++++
.../MCTargetDesc/AArch64MCLFIRewriter.cpp | 32 ++++++++
.../MCTargetDesc/AArch64MCLFIRewriter.h | 7 ++
.../AArch64/Atomics/atomic-store-pseudo.mir | 54 ++++++++++++
llvm/test/MC/AArch64/LFI/atomic-store-hint.s | 62 ++++++++++++++
12 files changed, 373 insertions(+), 72 deletions(-)
create mode 100644 clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
create mode 100644 llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
create mode 100644 llvm/test/MC/AArch64/LFI/atomic-store-hint.s
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 4b3c7d2e409b7..52a7564789fb7 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -2076,9 +2076,16 @@ static Value *EmitAtomicStoreWithHintBuiltin(CodeGenFunction &CGF,
llvm_unreachable(
"Expected integer policy argument to atomic store with hint.");
- StoreInst *Store =
- Builder.CreateStore(CGF.EmitScalarExpr(E->getArg(1)), // Value
- CGF.EmitPointerWithAlignment(E->getArg(0))); // Ptr;
+ const Expr *Ptr = E->getArg(0);
+ Address Addr = CGF.EmitPointerWithAlignment(Ptr);
+ Addr = Addr.withElementType(
+ CGF.ConvertTypeForMem(Ptr->getType()->getPointeeType()));
+
+ const Expr *Data = E->getArg(1);
+ Value *DataVal = CGF.EmitToMemory(CGF.EmitScalarExpr(Data), Data->getType());
+
+ StoreInst *Store = Builder.CreateStore(DataVal, Addr);
+ Store->setVolatile(Ptr->getType()->getPointeeType().isVolatileQualified());
AtomicOrdering Ordering;
unsigned OrderingArg = Result.Val.getInt().getExtValue();
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 1159b527c9a9e..0e8e0dac6523e 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -341,14 +341,26 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_builtin_must_be_pointer)
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
- QualType PtrQT = PtrTy->getPointeeType();
+ TheCall->setArg(0, PtrArg);
+ QualType PtrQT =
+ Context.getCanonicalType(PtrTy->getPointeeType()).getUnqualifiedType();
if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
!PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_op_needs_atomic_int_or_fp)
<< 0 << PtrQT << PtrArg->getSourceRange();
+ if (PtrQT->isBitIntType())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_builtin_bit_int_prohibit)
+ << PtrQT << PtrArg->getSourceRange();
+
+ if (Context.getCanonicalType(PtrTy->getPointeeType()).isConstQualified())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_non_const_pointer)
+ << PtrQT << PtrArg->getSourceRange();
+
unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
@@ -360,7 +372,8 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
if (DataArgRes.isInvalid())
return true;
- QualType DataQT = DataArgRes.get()->getType();
+ QualType DataQT = Context.getCanonicalType(DataArgRes.get()->getType())
+ .getUnqualifiedType();
if (PtrQT != DataQT)
return Diag(TheCall->getBeginLoc(),
@@ -370,42 +383,47 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// Arg 2 is the memory order, which must be relaxed, release or seq_cst
auto MemOrdArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)).get();
- std::optional<llvm::APSInt> MemOrdAP =
- MemOrdArg->getIntegerConstantExpr(Context);
- if (!MemOrdAP)
- return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_memory_order)
- << MemOrdArg->getType() << MemOrdArg->getSourceRange();
-
- unsigned Ordering = MemOrdAP->getZExtValue();
- if (!llvm::isValidAtomicOrderingCABI(Ordering))
- return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_memory_order)
- << *MemOrdAP << MemOrdArg->getSourceRange();
-
- auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
- if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
- AtomicOrdering != llvm::AtomicOrderingCABI::release &&
- AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
- return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_hint_has_invalid_memory_order)
- << *MemOrdAP << MemOrdArg->getSourceRange();
+ if (!MemOrdArg->isTypeDependent() && !MemOrdArg->isValueDependent()) {
+ std::optional<llvm::APSInt> MemOrdAP =
+ MemOrdArg->getIntegerConstantExpr(Context);
+ if (!MemOrdAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << MemOrdArg->getType() << MemOrdArg->getSourceRange();
+
+ unsigned Ordering = MemOrdAP->getZExtValue();
+ if (!llvm::isValidAtomicOrderingCABI(Ordering))
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+
+ auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
+ if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::release &&
+ AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_memory_order)
+ << *MemOrdAP << MemOrdArg->getSourceRange();
+ }
// Arg 3 is the hint type. Only values represented by AArch64MemoryHint
// are valid.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
- std::optional<llvm::APSInt> HintAP = HintArg->getIntegerConstantExpr(Context);
- if (!HintAP) {
- Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
- << HintArg->getType() << HintArg->getSourceRange();
- return false;
- }
+ if (!HintArg->isTypeDependent() && !HintArg->isValueDependent()) {
+ std::optional<llvm::APSInt> HintAP =
+ HintArg->getIntegerConstantExpr(Context);
+ if (!HintAP) {
+ Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
+ << HintArg->getType() << HintArg->getSourceRange();
+ return false;
+ }
- unsigned Hint = HintAP->getZExtValue();
- if (llvm::toAArch64MemoryHint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
- Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
- << *HintAP << HintArg->getSourceRange();
+ unsigned Hint = HintAP->getZExtValue();
+ if (llvm::toAArch64MemoryHint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
+ Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
+ << *HintAP << HintArg->getSourceRange();
+ }
return false;
}
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
new file mode 100644
index 0000000000000..99ec9816b4503
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -0,0 +1,38 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 6
+// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-elf -target-feature +v9.4a -target-feature +crc -target-feature +rand -target-feature +d128 -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+
+#include <arm_acle.h>
+
+// REQUIRES: aarch64-registered-target
+
+template <unsigned Order>
+void test_atomic_store_hint_template_order(int *ptr, int val) {
+// CHECK-LABEL: define weak_odr void @_Z37test_atomic_store_hint_template_orderILj3EEvPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] release, align 4, !mem.cache_hint [[META4:![0-9]+]]
+// CHECK-NEXT: ret void
+//
+ __builtin_arm_atomic_store_with_hint(ptr, val, Order, 0);
+}
+
+template void test_atomic_store_hint_template_order<__ATOMIC_RELEASE>(int *, int);
+
+template <unsigned Hint>
+// CHECK-LABEL: define weak_odr void @_Z36test_atomic_store_hint_template_hintILj1EEvPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] seq_cst, align 4, !mem.cache_hint [[META6:![0-9]+]]
+// CHECK-NEXT: ret void
+void test_atomic_store_hint_template_hint(int *ptr, int val) {
+ __builtin_arm_atomic_store_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
+}
+
+template void test_atomic_store_hint_template_hint<1>(int *, int);
+
+//.
+// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
+// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
+// CHECK: [[META6]] = !{i32 1, [[META7:![0-9]+]]}
+// CHECK: [[META7]] = !{!"aarch64.mem_hint", i32 1}
+//.
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index e8e9d4ee72534..a8cf697fb0bd6 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -8,6 +8,7 @@
#include <arm_acle.h>
+#include <stdbool.h>
// REQUIRES: arm-registered-target,aarch64-registered-target
@@ -1791,6 +1792,18 @@ void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
}
+// AArch64-LABEL: @test_atomic_store_hint_bool(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STOREDV:%.*]] = zext i1 [[DATA:%.*]] to i8
+// AArch64-NEXT: [[LOADEDV:%.*]] = icmp ne i8 [[STOREDV]], 0
+// AArch64-NEXT: [[STOREDV1:%.*]] = zext i1 [[LOADEDV]] to i8
+// AArch64-NEXT: store atomic i8 [[STOREDV1]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_bool(bool *ptr, bool data) {
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+}
+
// AArch64-LABEL: @test_atomic_store_hint_mfloat(
// AArch64-NEXT: entry:
// AArch64-NEXT: store atomic <1 x i8> [[DATA:%.*]], ptr [[PTR:%.*]] monotonic, align 1, !mem.cache_hint [[HINT1]]
@@ -1935,6 +1948,47 @@ void test_atomic_store_hint_double(double *ptr, double data) {
__arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
}
+typedef int aliased_int;
+
+// AArch64-LABEL: @test_atomic_store_hint_typedef_ptr(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] monotonic, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_typedef_ptr(aliased_int *ptr, int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_typedef_val(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic i32 [[VALUE:%.*]], ptr [[PTR:%.*]] release, align 4, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_typedef_val(int *ptr, aliased_int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELEASE, 1);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_volatile(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: store atomic volatile i32 [[VALUE:%.*]], ptr [[PTR:%.*]] seq_cst, align 4, !mem.cache_hint [[HINT1]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_volatile(volatile int *ptr, int value) {
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_SEQ_CST, 0);
+}
+
+// AArch64-LABEL: @test_atomic_store_hint_array_arg(
+// AArch64-NEXT: entry:
+// AArch64-NEXT: [[STORAGE:%.*]] = alloca [1 x i32], align 4
+// AArch64-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [1 x i32], ptr [[STORAGE]], i64 0, i64 0
+// AArch64-NEXT: store atomic i32 0, ptr [[ARRAYDECAY]] monotonic, align 4, !mem.cache_hint [[HINT3]]
+// AArch64-NEXT: ret void
+//
+void test_atomic_store_hint_array_arg() {
+ int storage[1];
+ __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, 1);
+}
+
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
// AArch64-NEXT: [[HINT2]] = !{!"aarch64.mem_hint", i32 0}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index c665507220a67..a010c6e9df285 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -54,7 +54,8 @@ void test_trap(short s, unsigned short us) {
void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
char c_data, __int128 inv_data, float f_data,
- int inv_int) {
+ int inv_int, const char *const_c_ptr,
+ unsigned _BitInt(7) *bit_ptr, unsigned _BitInt(7) *bit_data) {
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0); // expected-error {{too few arguments to function call, expected 4, have 3}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 0, 0); // expected-error {{too many arguments to function call, expected 4, have 5}}
@@ -62,6 +63,10 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
__builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
+ __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, 0); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('char' invalid)}}
+
+ __builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, 0); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 4d1932183cacb..5a502f785e0f4 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3281,8 +3281,8 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
MCInst Store;
Store.setOpcode(StOpc);
- Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
Store.setFlags(MI->getFlags());
if (Relaxed)
Store.addOperand(MCOperand::createImm(0));
@@ -3341,9 +3341,9 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
MCInst Store;
Store.setOpcode(StOpc);
- Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm
+ Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed
Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift
Store.setFlags(MI->getFlags());
@@ -3402,8 +3402,8 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
MCInst Store;
Store.setOpcode(StOpc);
- Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
+ Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm
Store.setFlags(MI->getFlags());
EmitToStreamer(*OutStreamer, Store);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index e0e65c400a8b0..7e0e75c7a3c6b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -288,23 +288,23 @@ def : Pat<(relaxed_store<atomic_store_64>
let Size = 8, isCodeGenOnly = 1, hasSideEffects = 1, mayStore = 1 in {
class BaseStoreHintPseudo<RegisterClass regtype>
- : Pseudo<(outs), (ins GPR64sp:$addr, regtype:$data,
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$addr,
i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROW<RegisterClass regtype, Operand WExt>
- : Pseudo<(outs), (ins GPR64sp:$Rn, GPR32:$Rm, regtype:$data, WExt:$extend,
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR32:$Rm, WExt:$extend,
i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoROX<RegisterClass regtype, Operand XExt>
- : Pseudo<(outs), (ins GPR64sp:$Rn, GPR64:$Rm, regtype:$data, XExt:$extend,
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, GPR64:$Rm, XExt:$extend,
i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoUImm<RegisterClass regtype, Operand UImmOp>
- : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, UImmOp:$offset,
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, UImmOp:$offset,
i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
class BaseStoreHintPseudoImm<RegisterClass regtype>
- : Pseudo<(outs), (ins GPR64sp:$Rn, regtype:$data, simm9:$offset,
+ : Pseudo<(outs), (ins regtype:$data, GPR64sp:$Rn, simm9:$offset,
i32imm:$relaxed, i32imm:$hint), []>, Sched<[WriteAtomic]>;
}
@@ -336,90 +336,90 @@ class atomic_hint_store<PatFrag Base, int Relaxed, code Pred>
multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Windexed8 GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend)),
- (ATOMIC_STORE_HINT_BroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend8:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_BroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend8:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
- (ATOMIC_STORE_HINT_HroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_HroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
- (ATOMIC_STORE_HINT_WroW GPR64sp:$addr, GPR32:$Rm, GPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_WroW GPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
- (ATOMIC_STORE_HINT_XroW GPR64sp:$addr, GPR32:$Rm, GPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_XroW GPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
(i32 (bitconvert (f32 FPR32Op:$data))), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
- (ATOMIC_STORE_HINT_SroW GPR64sp:$addr, GPR32:$Rm, FPR32:$data, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_SroW FPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
- (ATOMIC_STORE_HINT_DroW GPR64sp:$addr, GPR32:$Rm, FPR64:$data, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_DroW FPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
- (ATOMIC_STORE_HINT_BroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend8:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_BroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
- (ATOMIC_STORE_HINT_HroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_HroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
- (ATOMIC_STORE_HINT_WroX GPR64sp:$addr, GPR64:$Rm, GPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_WroX GPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
- (ATOMIC_STORE_HINT_XroX GPR64sp:$addr, GPR64:$Rm, GPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_XroX GPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
(i32 (bitconvert (f32 FPR32Op:$data))), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
- (ATOMIC_STORE_HINT_SroX GPR64sp:$addr, GPR64:$Rm, FPR32:$data, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_SroX FPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
- (ATOMIC_STORE_HINT_DroX GPR64sp:$addr, GPR64:$Rm, FPR64:$data, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_DroX FPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
- (ATOMIC_STORE_HINT_Bui GPR64sp:$addr, GPR32:$data, uimm12s1:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Bui GPR32:$data, GPR64sp:$addr, uimm12s1:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
- (ATOMIC_STORE_HINT_Hui GPR64sp:$addr, GPR32:$data, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Hui GPR32:$data, GPR64sp:$addr, uimm12s2:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
- (ATOMIC_STORE_HINT_Wui GPR64sp:$addr, GPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Wui GPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
- (ATOMIC_STORE_HINT_Xui GPR64sp:$addr, GPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Xui GPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
(i32 (bitconvert (f32 FPR32Op:$data))), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
- (ATOMIC_STORE_HINT_Sui GPR64sp:$addr, FPR32:$data, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Sui FPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
- (ATOMIC_STORE_HINT_Dui GPR64sp:$addr, FPR64:$data, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Dui FPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Bi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Bi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_16, 1, Pred>
GPR32:$data, (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Hi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Hi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
GPR32:$data, (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Wi GPR64sp:$addr, GPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Wi GPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
GPR64:$data, (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Xi GPR64sp:$addr, GPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Xi GPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, 1, Pred>
(i32 (bitconvert (f32 FPR32Op:$data))), (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Si GPR64sp:$addr, FPR32:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Si FPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
- (ATOMIC_STORE_HINT_Di GPR64sp:$addr, FPR64:$data, simm9:$offset, (i32 1), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_Di FPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
}
multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_8, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_B GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_B GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_16, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_H GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_H GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_32, Relaxed, Pred> GPR32:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_W GPR64sp:$addr, GPR32:$data, (i32 Relaxed), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_W GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr),
- (ATOMIC_STORE_HINT_X GPR64sp:$addr, GPR64:$data, (i32 Relaxed), (i32 Hint))>;
+ (ATOMIC_STORE_HINT_X GPR64:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
}
let AddedComplexity = 15 in {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index b8e74f8ad496d..6962d66dedaf0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3151,6 +3151,18 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
case AArch64::STZ2Gi:
case AArch64::STZGi:
case AArch64::TAGPstack:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_Di:
+ case AArch64::ATOMIC_STORE_HINT_Bui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
return 2;
case AArch64::LD1B_D_IMM:
case AArch64::LD1B_H_IMM:
@@ -4761,6 +4773,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STRXui:
case AArch64::STRDui:
case AArch64::PRFMui:
+ case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_Dui:
Scale = Width = TypeSize::getFixed(8);
MinOffset = 0;
MaxOffset = 4095;
@@ -4770,6 +4784,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSWui:
case AArch64::STRWui:
case AArch64::STRSui:
+ case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_Sui:
Scale = Width = TypeSize::getFixed(4);
MinOffset = 0;
MaxOffset = 4095;
@@ -4780,6 +4796,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSHXui:
case AArch64::STRHui:
case AArch64::STRHHui:
+ case AArch64::ATOMIC_STORE_HINT_Hui:
Scale = Width = TypeSize::getFixed(2);
MinOffset = 0;
MaxOffset = 4095;
@@ -4790,6 +4807,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::LDRSBXui:
case AArch64::STRBui:
case AArch64::STRBBui:
+ case AArch64::ATOMIC_STORE_HINT_Bui:
Scale = Width = TypeSize::getFixed(1);
MinOffset = 0;
MaxOffset = 4095;
@@ -4868,6 +4886,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURDi:
case AArch64::STLURXi:
case AArch64::PRFUMi:
+ case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_Di:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(8);
MinOffset = -256;
@@ -4881,6 +4901,8 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURWi:
case AArch64::STURSi:
case AArch64::STLURWi:
+ case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_Si:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(4);
MinOffset = -256;
@@ -4896,6 +4918,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURHi:
case AArch64::STURHHi:
case AArch64::STLURHi:
+ case AArch64::ATOMIC_STORE_HINT_Hi:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(2);
MinOffset = -256;
@@ -4911,6 +4934,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURBi:
case AArch64::STURBBi:
case AArch64::STLURBi:
+ case AArch64::ATOMIC_STORE_HINT_Bi:
Scale = Width = TypeSize::getFixed(1);
MinOffset = -256;
MaxOffset = 255;
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
index e6f55edd45239..bd8d9d6db22f5 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
@@ -158,6 +158,13 @@ static bool mayPrefetch(const MCInst &Inst) {
}
}
+// Memory hints 48 (stshh keep) & 49 (stshh strm) should be emitted
+// immediately before a store instruction.
+static bool isMemHint(const MCInst &Inst) {
+ return Inst.getOpcode() == AArch64::HINT && Inst.getOperand(0).isImm() &&
+ (Inst.getOperand(0).getImm() > 47 && Inst.getOperand(0).getImm() < 50);
+}
+
static bool isAuthenticatedBranch(unsigned Opcode) {
switch (Opcode) {
case AArch64::BRAA:
@@ -320,10 +327,17 @@ void AArch64MCLFIRewriter::finish(MCStreamer &Out) {
emitAddMask(AArch64::LR, AArch64::LR, Out, *LastSTI);
DeferredLRGuard = false;
}
+
+ // Flush a deferred memory hint instruction.
+ if (PendingMemHintInst && LastSTI)
+ emitPendingMemHintInst(Out, *LastSTI);
}
void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI) {
+ if (!isMemHint(Inst) && mayStore(Inst))
+ emitPendingMemHintInst(Out, STI);
+
// Invalidate the active guard if this instruction modifies the guarded
// register, modifies x28 itself, or may affect control flow.
if (ActiveGuardReg) {
@@ -374,6 +388,14 @@ void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
emitInst(MCInstBuilder(AArch64::TLSDESCCALL).addExpr(Expr), Out, STI);
}
+void AArch64MCLFIRewriter::emitPendingMemHintInst(MCStreamer &Out,
+ const MCSubtargetInfo &STI) {
+ if (!PendingMemHintInst)
+ return;
+ emitInst(*PendingMemHintInst, Out, STI);
+ PendingMemHintInst.reset();
+}
+
void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
MCStreamer &Out,
const MCSubtargetInfo &STI) {
@@ -875,6 +897,10 @@ void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI) {
+ // Emit any pending hint if the next instruction is not a store.
+ if (!isMemHint(Inst) && !mayStore(Inst))
+ emitPendingMemHintInst(Out, STI);
+
if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
PendingTLSDescCall = Inst.getOperand(0).getExpr();
return;
@@ -955,6 +981,12 @@ void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
(mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
return rewriteLoadStore(Inst, Out, STI);
+ // Defer stshh hint instructions until rewriting the store.
+ if (isMemHint(Inst)) {
+ PendingMemHintInst = Inst;
+ return;
+ }
+
emitInst(Inst, Out, STI);
}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
index d7a70b0e163a5..ca4c42a235ac4 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
@@ -14,6 +14,7 @@
#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCLFIREWRITER_H
#include "AArch64AddressingModes.h"
+#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCLFIRewriter.h"
#include "llvm/MC/MCRegister.h"
@@ -75,6 +76,11 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
/// the guard and the branch so the relocation stays on the BLR.
const MCExpr *PendingTLSDescCall = nullptr;
+ // Deferred memory hint instruction (e.g. `stshh keep`). This instruction
+ // must be emitted immediately before a store and is deferred until after
+ // LFI inserts any guard instructions.
+ std::optional<MCInst> PendingMemHintInst;
+
/// Rewriter state for implementing the guard-elimination optimization, which
/// allows redundant add masks to be skipped. When it holds a value, x28 is
/// known to already hold the guarded value of that register.
@@ -93,6 +99,7 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
void emitBranch(unsigned Opcode, MCRegister Target, MCStreamer &Out,
const MCSubtargetInfo &STI);
void emitPendingTLSDescCall(MCStreamer &Out, const MCSubtargetInfo &STI);
+ void emitPendingMemHintInst(MCStreamer &Out, const MCSubtargetInfo &STI);
void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out,
const MCSubtargetInfo &STI);
void emitAddImm(MCRegister Dest, MCRegister Src, int64_t Imm, MCStreamer &Out,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
new file mode 100644
index 0000000000000..31f82a33de8eb
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/atomic-store-pseudo.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -run-pass=prolog-epilog -simplify-mir -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+ define void @test_atomic_store_pseudo() nounwind { entry: unreachable }
+...
+---
+name: test_atomic_store_pseudo
+tracksRegLiveness: true
+liveins:
+ - { reg: '$w0' }
+ - { reg: '$s0' }
+ - { reg: '$x0' }
+ - { reg: '$d0' }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $w0, $s0, $x0, $d0
+
+ ; CHECK-LABEL: name: test_atomic_store_pseudo
+ ; CHECK: liveins: $w0, $s0, $x0, $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sp = frame-setup SUBXri $sp, 16, 0
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Bi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Hi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Wi $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Si $s0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Xi $x0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Di $d0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Bui $w0, $sp, 8, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Hui $w0, $sp, 4, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Wui $w0, $sp, 2, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Sui $s0, $sp, 2, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Xui $x0, $sp, 1, 1, 48
+ ; CHECK-NEXT: ATOMIC_STORE_HINT_Dui $d0, $sp, 1, 1, 48
+ ; CHECK-NEXT: $sp = frame-destroy ADDXri $sp, 16, 0
+ ; CHECK-NEXT: RET_ReallyLR
+ ATOMIC_STORE_HINT_Bi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Hi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Wi $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Si $s0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Xi $x0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Di $d0, %stack.0, 0, 1, 48
+
+ ATOMIC_STORE_HINT_Bui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Hui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Wui $w0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Sui $s0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Xui $x0, %stack.0, 0, 1, 48
+ ATOMIC_STORE_HINT_Dui $d0, %stack.0, 0, 1, 48
+
+ RET_ReallyLR
+...
diff --git a/llvm/test/MC/AArch64/LFI/atomic-store-hint.s b/llvm/test/MC/AArch64/LFI/atomic-store-hint.s
new file mode 100644
index 0000000000000..fedbc8fbba864
--- /dev/null
+++ b/llvm/test/MC/AArch64/LFI/atomic-store-hint.s
@@ -0,0 +1,62 @@
+// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
+
+// Ensure hint instruction is emitted immediately before store
+
+stshh keep
+stlr w1, [x0]
+// CHECK: add x28, x27, w0, uxtw
+// CHECK-NEXT: stshh keep
+// CHECK-NEXT: stlr w1, [x28]
+
+stshh strm
+strb w2, [x0, w1, sxtw]
+// CHECK: add x26, x0, w1, sxtw
+// CHECK-NEXT: stshh strm
+// CHECK-NEXT: strb w2, [x27, w26, uxtw]
+
+stshh keep
+stur w0, [x1, #-256]
+// CHECK: add x28, x27, w1, uxtw
+// CHECK-NEXT: stshh keep
+// CHECK-NEXT: stur w0, [x28, #-256]
+
+stshh strm
+strh w2, [x0, x1, lsl #1]
+// CHECK: add x26, x0, x1, lsl #1
+// CHECK-NEXT: stshh strm
+// CHECK-NEXT: strh w2, [x27, w26, uxtw]
+
+stshh keep
+label_after_hint:
+stlr w1, [x0]
+// CHECK: label_after_hint:
+// CHECK-NEXT: add x28, x27, w0, uxtw
+// CHECK-NEXT: stshh keep
+// CHECK-NEXT: stlr w1, [x28]
+
+// The hint should always appear before a store instruction, but
+// test that we do not change the order if this was not the case.
+
+stshh keep
+b label_branch
+label_branch:
+stlr w1, [x0]
+// CHECK: stshh keep
+// CHECK-NEXT: b label_branch
+// CHECK-NEXT: label_branch:
+// CHECK-NEXT: add x28, x27, w0, uxtw
+// CHECK-NEXT: stlr w1, [x28]
+
+stshh keep
+ldp x0, x1, [x2]
+// CHECK: stshh keep
+// CHECK-NEXT: add x28, x27, w2, uxtw
+// CHECK-NEXT: ldp x0, x1, [x28]
+
+stshh strm
+add x0, x0, #1
+stlr w1, [x0]
+// CHECK: stshh strm
+// CHECK-NEXT: add x0, x0, #1
+// CHECK-NEXT: add x28, x27, w0, uxtw
+// CHECK-NEXT: stlr w1, [x28]
>From cc6b4a7d623b5d1a871a02f323d716a0c88c2ef5 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 10 Sep 2026 15:07:55 +0000
Subject: [PATCH 10/13] - Remove LFI changes - Simplify RUN line for
atomic-store-hint-template.cpp - Remove isTypeDependent() checks
---
clang/lib/Sema/SemaARM.cpp | 18 +++---
.../AArch64/atomic-store-hint-template.cpp | 2 +-
clang/test/Sema/builtins-arm64.c | 2 +-
.../MCTargetDesc/AArch64MCLFIRewriter.cpp | 32 ----------
.../MCTargetDesc/AArch64MCLFIRewriter.h | 7 ---
llvm/test/MC/AArch64/LFI/atomic-store-hint.s | 62 -------------------
6 files changed, 11 insertions(+), 112 deletions(-)
delete mode 100644 llvm/test/MC/AArch64/LFI/atomic-store-hint.s
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index 0e8e0dac6523e..f4a37d4469e2b 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -343,8 +343,13 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
<< PtrArg->getType() << 0 << PtrArg->getSourceRange();
TheCall->setArg(0, PtrArg);
- QualType PtrQT =
- Context.getCanonicalType(PtrTy->getPointeeType()).getUnqualifiedType();
+ QualType PtrQT = Context.getCanonicalType(PtrTy->getPointeeType());
+ if (PtrQT.isConstQualified())
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_op_needs_non_const_pointer)
+ << PtrQT << PtrArg->getSourceRange();
+
+ PtrQT = PtrQT.getUnqualifiedType();
if (!PtrQT->isIntegralType(Context) && !PtrQT->isFloatingType() &&
!PtrQT->isMFloat8Type())
return Diag(TheCall->getBeginLoc(),
@@ -356,11 +361,6 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
diag::err_atomic_builtin_bit_int_prohibit)
<< PtrQT << PtrArg->getSourceRange();
- if (Context.getCanonicalType(PtrTy->getPointeeType()).isConstQualified())
- return Diag(TheCall->getBeginLoc(),
- diag::err_atomic_op_needs_non_const_pointer)
- << PtrQT << PtrArg->getSourceRange();
-
unsigned TySize = Context.getTypeSize(PtrQT);
if (TySize != 8 && TySize != 16 && TySize != 32 && TySize != 64)
return Diag(TheCall->getBeginLoc(), diag::err_atomic_op_hint_data_size)
@@ -383,7 +383,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// Arg 2 is the memory order, which must be relaxed, release or seq_cst
auto MemOrdArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)).get();
- if (!MemOrdArg->isTypeDependent() && !MemOrdArg->isValueDependent()) {
+ if (!MemOrdArg->isValueDependent()) {
std::optional<llvm::APSInt> MemOrdAP =
MemOrdArg->getIntegerConstantExpr(Context);
if (!MemOrdAP)
@@ -410,7 +410,7 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// are valid.
auto HintArg =
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
- if (!HintArg->isTypeDependent() && !HintArg->isValueDependent()) {
+ if (!HintArg->isValueDependent()) {
std::optional<llvm::APSInt> HintAP =
HintArg->getIntegerConstantExpr(Context);
if (!HintAP) {
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
index 99ec9816b4503..afac4ab15b15c 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -1,5 +1,5 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 6
-// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-elf -target-feature +v9.4a -target-feature +crc -target-feature +rand -target-feature +d128 -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
+// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-elf -target-feature +v9.4a -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
#include <arm_acle.h>
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index a010c6e9df285..1bc24f2268fab 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -63,7 +63,7 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
__builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
- __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, 0); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('char' invalid)}}
+ __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, 0); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
__builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, 0); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
index bd8d9d6db22f5..e6f55edd45239 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.cpp
@@ -158,13 +158,6 @@ static bool mayPrefetch(const MCInst &Inst) {
}
}
-// Memory hints 48 (stshh keep) & 49 (stshh strm) should be emitted
-// immediately before a store instruction.
-static bool isMemHint(const MCInst &Inst) {
- return Inst.getOpcode() == AArch64::HINT && Inst.getOperand(0).isImm() &&
- (Inst.getOperand(0).getImm() > 47 && Inst.getOperand(0).getImm() < 50);
-}
-
static bool isAuthenticatedBranch(unsigned Opcode) {
switch (Opcode) {
case AArch64::BRAA:
@@ -327,17 +320,10 @@ void AArch64MCLFIRewriter::finish(MCStreamer &Out) {
emitAddMask(AArch64::LR, AArch64::LR, Out, *LastSTI);
DeferredLRGuard = false;
}
-
- // Flush a deferred memory hint instruction.
- if (PendingMemHintInst && LastSTI)
- emitPendingMemHintInst(Out, *LastSTI);
}
void AArch64MCLFIRewriter::emitInst(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI) {
- if (!isMemHint(Inst) && mayStore(Inst))
- emitPendingMemHintInst(Out, STI);
-
// Invalidate the active guard if this instruction modifies the guarded
// register, modifies x28 itself, or may affect control flow.
if (ActiveGuardReg) {
@@ -388,14 +374,6 @@ void AArch64MCLFIRewriter::emitPendingTLSDescCall(MCStreamer &Out,
emitInst(MCInstBuilder(AArch64::TLSDESCCALL).addExpr(Expr), Out, STI);
}
-void AArch64MCLFIRewriter::emitPendingMemHintInst(MCStreamer &Out,
- const MCSubtargetInfo &STI) {
- if (!PendingMemHintInst)
- return;
- emitInst(*PendingMemHintInst, Out, STI);
- PendingMemHintInst.reset();
-}
-
void AArch64MCLFIRewriter::emitMov(MCRegister Dest, MCRegister Src,
MCStreamer &Out,
const MCSubtargetInfo &STI) {
@@ -897,10 +875,6 @@ void AArch64MCLFIRewriter::rewriteVASysOp(const MCInst &Inst, MCStreamer &Out,
// AArch64InstrInfo::getLFIInstSizeInBytes must be updated to match.
void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
const MCSubtargetInfo &STI) {
- // Emit any pending hint if the next instruction is not a store.
- if (!isMemHint(Inst) && !mayStore(Inst))
- emitPendingMemHintInst(Out, STI);
-
if (Inst.getOpcode() == AArch64::TLSDESCCALL) {
PendingTLSDescCall = Inst.getOperand(0).getExpr();
return;
@@ -981,12 +955,6 @@ void AArch64MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
(mayLoad(Inst) || mayStore(Inst) || mayPrefetch(Inst)))
return rewriteLoadStore(Inst, Out, STI);
- // Defer stshh hint instructions until rewriting the store.
- if (isMemHint(Inst)) {
- PendingMemHintInst = Inst;
- return;
- }
-
emitInst(Inst, Out, STI);
}
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
index ca4c42a235ac4..d7a70b0e163a5 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCLFIRewriter.h
@@ -14,7 +14,6 @@
#define LLVM_LIB_TARGET_AARCH64_MCTARGETDESC_AARCH64MCLFIREWRITER_H
#include "AArch64AddressingModes.h"
-#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCLFIRewriter.h"
#include "llvm/MC/MCRegister.h"
@@ -76,11 +75,6 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
/// the guard and the branch so the relocation stays on the BLR.
const MCExpr *PendingTLSDescCall = nullptr;
- // Deferred memory hint instruction (e.g. `stshh keep`). This instruction
- // must be emitted immediately before a store and is deferred until after
- // LFI inserts any guard instructions.
- std::optional<MCInst> PendingMemHintInst;
-
/// Rewriter state for implementing the guard-elimination optimization, which
/// allows redundant add masks to be skipped. When it holds a value, x28 is
/// known to already hold the guarded value of that register.
@@ -99,7 +93,6 @@ class AArch64MCLFIRewriter : public MCLFIRewriter {
void emitBranch(unsigned Opcode, MCRegister Target, MCStreamer &Out,
const MCSubtargetInfo &STI);
void emitPendingTLSDescCall(MCStreamer &Out, const MCSubtargetInfo &STI);
- void emitPendingMemHintInst(MCStreamer &Out, const MCSubtargetInfo &STI);
void emitMov(MCRegister Dest, MCRegister Src, MCStreamer &Out,
const MCSubtargetInfo &STI);
void emitAddImm(MCRegister Dest, MCRegister Src, int64_t Imm, MCStreamer &Out,
diff --git a/llvm/test/MC/AArch64/LFI/atomic-store-hint.s b/llvm/test/MC/AArch64/LFI/atomic-store-hint.s
deleted file mode 100644
index fedbc8fbba864..0000000000000
--- a/llvm/test/MC/AArch64/LFI/atomic-store-hint.s
+++ /dev/null
@@ -1,62 +0,0 @@
-// RUN: llvm-mc -triple aarch64_lfi %s | FileCheck %s
-
-// Ensure hint instruction is emitted immediately before store
-
-stshh keep
-stlr w1, [x0]
-// CHECK: add x28, x27, w0, uxtw
-// CHECK-NEXT: stshh keep
-// CHECK-NEXT: stlr w1, [x28]
-
-stshh strm
-strb w2, [x0, w1, sxtw]
-// CHECK: add x26, x0, w1, sxtw
-// CHECK-NEXT: stshh strm
-// CHECK-NEXT: strb w2, [x27, w26, uxtw]
-
-stshh keep
-stur w0, [x1, #-256]
-// CHECK: add x28, x27, w1, uxtw
-// CHECK-NEXT: stshh keep
-// CHECK-NEXT: stur w0, [x28, #-256]
-
-stshh strm
-strh w2, [x0, x1, lsl #1]
-// CHECK: add x26, x0, x1, lsl #1
-// CHECK-NEXT: stshh strm
-// CHECK-NEXT: strh w2, [x27, w26, uxtw]
-
-stshh keep
-label_after_hint:
-stlr w1, [x0]
-// CHECK: label_after_hint:
-// CHECK-NEXT: add x28, x27, w0, uxtw
-// CHECK-NEXT: stshh keep
-// CHECK-NEXT: stlr w1, [x28]
-
-// The hint should always appear before a store instruction, but
-// test that we do not change the order if this was not the case.
-
-stshh keep
-b label_branch
-label_branch:
-stlr w1, [x0]
-// CHECK: stshh keep
-// CHECK-NEXT: b label_branch
-// CHECK-NEXT: label_branch:
-// CHECK-NEXT: add x28, x27, w0, uxtw
-// CHECK-NEXT: stlr w1, [x28]
-
-stshh keep
-ldp x0, x1, [x2]
-// CHECK: stshh keep
-// CHECK-NEXT: add x28, x27, w2, uxtw
-// CHECK-NEXT: ldp x0, x1, [x28]
-
-stshh strm
-add x0, x0, #1
-stlr w1, [x0]
-// CHECK: stshh strm
-// CHECK-NEXT: add x0, x0, #1
-// CHECK-NEXT: add x28, x27, w0, uxtw
-// CHECK-NEXT: stlr w1, [x28]
>From 0d5f4fccf92698012f5fbf1c426d4049e5fab1f6 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Mon, 14 Sep 2026 14:09:13 +0000
Subject: [PATCH 11/13] - Allow use of HINT_STSHH_KEEP & HINT_STSHH_STRM with
intrinsic
---
clang/lib/Headers/arm_acle.h | 2 +
.../AArch64/atomic-store-hint-template.cpp | 2 +-
clang/test/CodeGen/arm_acle.c | 44 +++++++++----------
clang/test/CodeGen/builtins-arm64.c | 6 ++-
clang/test/Sema/builtins-arm64.c | 8 ++--
5 files changed, 35 insertions(+), 27 deletions(-)
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index 8d4fbdc2e8c32..715fe851b0c32 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -744,6 +744,8 @@ __arm_st64bv0(void *__addr, data512_t __value) {
/* Atomic store with hints */
#if defined(__ARM_64BIT_STATE) && __ARM_64BIT_STATE
+#define HINT_STSHH_KEEP 0
+#define HINT_STSHH_STRM 1
#define __arm_atomic_store_with_hint(ptr, data, memory_order, hint) \
__builtin_arm_atomic_store_with_hint(ptr, data, memory_order, hint)
#endif
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
index afac4ab15b15c..a5f3d416f61ad 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -28,7 +28,7 @@ void test_atomic_store_hint_template_hint(int *ptr, int val) {
__builtin_arm_atomic_store_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
}
-template void test_atomic_store_hint_template_hint<1>(int *, int);
+template void test_atomic_store_hint_template_hint<HINT_STSHH_STRM>(int *, int);
//.
// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index a8cf697fb0bd6..60ec3e48e9498 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -1780,7 +1780,7 @@ void test_atomic_store_hint_char(char *ptr, char data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_schar(
@@ -1789,7 +1789,7 @@ void test_atomic_store_hint_uchar(unsigned char *ptr, unsigned char data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_bool(
@@ -1801,7 +1801,7 @@ void test_atomic_store_hint_schar(signed char *ptr, signed char data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_bool(bool *ptr, bool data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_mfloat(
@@ -1810,7 +1810,7 @@ void test_atomic_store_hint_bool(bool *ptr, bool data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_mfloat(__mfp8 *ptr, __mfp8 data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_bfloat(
@@ -1828,7 +1828,7 @@ void test_atomic_store_hint_bfloat(__bf16 *ptr, __bf16 data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 1);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_STRM);
}
// AArch64-LABEL: @test_atomic_store_hint_short(
@@ -1837,7 +1837,7 @@ void test_atomic_store_hint_half(__fp16 *ptr, __fp16 data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_short(short *ptr, short data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_ushort(
@@ -1846,7 +1846,7 @@ void test_atomic_store_hint_short(short *ptr, short data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_int(
@@ -1855,7 +1855,7 @@ void test_atomic_store_hint_ushort(unsigned short *ptr, unsigned short data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_int(int *ptr, int data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_unsigned(
@@ -1864,7 +1864,7 @@ void test_atomic_store_hint_int(int *ptr, int data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_u32(
@@ -1873,7 +1873,7 @@ void test_atomic_store_hint_unsigned(unsigned *ptr, unsigned data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_s32(
@@ -1882,7 +1882,7 @@ void test_atomic_store_hint_u32(uint32_t *ptr, uint32_t data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_float(
@@ -1891,7 +1891,7 @@ void test_atomic_store_hint_s32(int32_t *ptr, int32_t data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_float(float *ptr, float data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_s64(
@@ -1900,7 +1900,7 @@ void test_atomic_store_hint_float(float *ptr, float data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM);
}
// AArch64-LABEL: @test_atomic_store_hint_long(
@@ -1909,7 +1909,7 @@ void test_atomic_store_hint_s64(int64_t *ptr, int64_t data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long(long *ptr, long data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_ulong(
@@ -1918,7 +1918,7 @@ void test_atomic_store_hint_long(long *ptr, long data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_long_long_int(
@@ -1927,7 +1927,7 @@ void test_atomic_store_hint_ulong(unsigned long *ptr, unsigned long data) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_long_long_uint(
@@ -1936,7 +1936,7 @@ void test_atomic_store_hint_long_long_int(long long int *ptr, long long int data
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned long long int data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, 0);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELEASE, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_double(
@@ -1945,7 +1945,7 @@ void test_atomic_store_hint_long_long_uint(unsigned long long int *ptr, unsigned
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_double(double *ptr, double data) {
- __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, 1);
+ __arm_atomic_store_with_hint(ptr, data, __ATOMIC_RELAXED, HINT_STSHH_STRM);
}
typedef int aliased_int;
@@ -1956,7 +1956,7 @@ typedef int aliased_int;
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_typedef_ptr(aliased_int *ptr, int value) {
- __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, 0);
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_typedef_val(
@@ -1965,7 +1965,7 @@ void test_atomic_store_hint_typedef_ptr(aliased_int *ptr, int value) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_typedef_val(int *ptr, aliased_int value) {
- __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELEASE, 1);
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELEASE, HINT_STSHH_STRM);
}
// AArch64-LABEL: @test_atomic_store_hint_volatile(
@@ -1974,7 +1974,7 @@ void test_atomic_store_hint_typedef_val(int *ptr, aliased_int value) {
// AArch64-NEXT: ret void
//
void test_atomic_store_hint_volatile(volatile int *ptr, int value) {
- __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_SEQ_CST, 0);
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
}
// AArch64-LABEL: @test_atomic_store_hint_array_arg(
@@ -1986,7 +1986,7 @@ void test_atomic_store_hint_volatile(volatile int *ptr, int value) {
//
void test_atomic_store_hint_array_arg() {
int storage[1];
- __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, 1);
+ __builtin_arm_atomic_store_with_hint(storage, 0, __ATOMIC_RELAXED, HINT_STSHH_STRM);
}
// AArch64: [[HINT1]] = !{i32 1, [[HINT2:![0-9]+]]}
diff --git a/clang/test/CodeGen/builtins-arm64.c b/clang/test/CodeGen/builtins-arm64.c
index 672dbe016a1cd..fb297cd3eec2c 100644
--- a/clang/test/CodeGen/builtins-arm64.c
+++ b/clang/test/CodeGen/builtins-arm64.c
@@ -2,6 +2,7 @@
// RUN: %clang_cc1 -triple aarch64-windows -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-WIN
// RUN: %clang_cc1 -triple arm64_32-apple-ios13 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
#include <stdint.h>
+#include <arm_acle.h>
void f0(void *a, void *b) {
__clear_cache(a,b);
@@ -220,12 +221,15 @@ void atomic_store_with_hint(int64_t *a, int64_t b) {
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 0); // HINT_STSHH_KEEP
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8, !mem.cache_hint ![[M1:[0-9]]]
- __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, 0);
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_SEQ_CST, HINT_STSHH_KEEP);
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} seq_cst, align 8, !mem.cache_hint ![[M1]]
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, 1); // HINT_STSHH_STRM
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
+ __builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELEASE, HINT_STSHH_STRM);
+ // CHECK: store atomic i64 {{.*}}, ptr {{.*}} release, align 8, !mem.cache_hint ![[M3:[0-9]+]]
+
// Invalid hint should be dropped
__builtin_arm_atomic_store_with_hint(a, b, __ATOMIC_RELAXED, 2); // Invalid Hint
// CHECK: store atomic i64 {{.*}}, ptr {{.*}} monotonic, align 8
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 1bc24f2268fab..5a14deb295a78 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -1,6 +1,8 @@
// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify %s
// RUN: %clang_cc1 -triple arm64-apple-ios -DTEST1 -fsyntax-only -verify %s
+#include <arm_acle.h>
+
#ifdef TEST1
void __clear_cache(void *start, void *end);
#endif
@@ -63,15 +65,15 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, f_data, 0, 0); // expected-error {{arguments are of different types ('char' vs 'float')}}
__builtin_arm_atomic_store_with_hint(inv_ptr, inv_data, 0, 0); // expected-error {{address argument to atomic store with hint must be of size 8, 16, 32 or 64 bits}}
- __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, 0); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
+ __builtin_arm_atomic_store_with_hint(const_c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_KEEP); // expected-error {{address argument to atomic operation must be a pointer to non-const type ('const char' invalid)}}
- __builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, 0); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+ __builtin_arm_atomic_store_with_hint(bit_ptr, bit_data, __ATOMIC_RELAXED, HINT_STSHH_STRM); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}}
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, __ATOMIC_RELAXED, HINT_STSHH_INVALID); // expected-error {{use of undeclared identifier 'HINT_STSHH_INVALID'}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, inv_int, 0); // expected-error {{invalid memory order argument to atomic hint operation ('int' invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 2, 0); // expected-error {{invalid memory order argument to atomic hint operation (2 invalid)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
-
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{invalid hint type argument to atomic hint operation ('char *')}}
}
>From 8438b4554784295969401ff0479b16529e047493 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Wed, 16 Sep 2026 08:46:58 +0000
Subject: [PATCH 12/13] - Use convertArgumentToType & store converted values to
call argument list - Remove unrelated change from visitAtomicStore
---
clang/include/clang/Sema/Sema.h | 2 +
clang/lib/Sema/SemaARM.cpp | 57 ++++++++++++-------
clang/lib/Sema/SemaChecking.cpp | 12 ++--
.../AArch64/atomic-store-hint-template.cpp | 29 +++++++++-
clang/test/Sema/builtins-arm64.c | 4 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 4 +-
6 files changed, 76 insertions(+), 32 deletions(-)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5f24adbf9e9cf..eb3b908351448 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2915,6 +2915,8 @@ class Sema final : public SemaBase {
/// This is useful when doing custom type-checking. Returns true on error.
bool checkArgCount(CallExpr *Call, unsigned DesiredArgCount);
+ bool convertArgumentToType(Expr *&Value, QualType Ty);
+
/// Returns true if the argument consists of one contiguous run of 1s with any
/// number of 0s on either side. The 1s are allowed to wrap from LSB to MSB,
/// so 0x000FFF0, 0x0000FFFF, 0xFF0000FF, 0x0 are all runs. 0x0F0F0000 is not,
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f4a37d4469e2b..2bf6901bcc07f 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -372,8 +372,10 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(1));
if (DataArgRes.isInvalid())
return true;
- QualType DataQT = Context.getCanonicalType(DataArgRes.get()->getType())
- .getUnqualifiedType();
+ auto *DataArg = DataArgRes.get();
+ QualType DataQT =
+ Context.getCanonicalType(DataArg->getType()).getUnqualifiedType();
+ TheCall->setArg(1, DataArg);
if (PtrQT != DataQT)
return Diag(TheCall->getBeginLoc(),
@@ -382,20 +384,27 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
// Arg 2 is the memory order, which must be relaxed, release or seq_cst
auto MemOrdArg =
- SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2)).get();
- if (!MemOrdArg->isValueDependent()) {
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(2));
+ if (MemOrdArg.isInvalid())
+ return true;
+ auto *MemOrd = MemOrdArg.get();
+ if (SemaRef.convertArgumentToType(MemOrd, Context.IntTy))
+ return true;
+ TheCall->setArg(2, MemOrd);
+
+ if (!MemOrd->isValueDependent()) {
std::optional<llvm::APSInt> MemOrdAP =
- MemOrdArg->getIntegerConstantExpr(Context);
+ MemOrd->getIntegerConstantExpr(Context);
if (!MemOrdAP)
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_has_invalid_memory_order)
- << MemOrdArg->getType() << MemOrdArg->getSourceRange();
+ << MemOrd->getType() << MemOrd->getSourceRange();
unsigned Ordering = MemOrdAP->getZExtValue();
if (!llvm::isValidAtomicOrderingCABI(Ordering))
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_has_invalid_memory_order)
- << *MemOrdAP << MemOrdArg->getSourceRange();
+ << *MemOrdAP << MemOrd->getSourceRange();
auto AtomicOrdering = static_cast<llvm::AtomicOrderingCABI>(Ordering);
if (AtomicOrdering != llvm::AtomicOrderingCABI::relaxed &&
@@ -403,28 +412,34 @@ bool SemaARM::BuiltinARMAtomicStoreHintCall(unsigned BuiltinID,
AtomicOrdering != llvm::AtomicOrderingCABI::seq_cst)
return Diag(TheCall->getBeginLoc(),
diag::err_atomic_hint_has_invalid_memory_order)
- << *MemOrdAP << MemOrdArg->getSourceRange();
+ << *MemOrdAP << MemOrd->getSourceRange();
}
// Arg 3 is the hint type. Only values represented by AArch64MemoryHint
// are valid.
auto HintArg =
- SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3)).get();
- if (!HintArg->isValueDependent()) {
- std::optional<llvm::APSInt> HintAP =
- HintArg->getIntegerConstantExpr(Context);
- if (!HintAP) {
- Diag(TheCall->getBeginLoc(), diag::err_atomic_hint_has_invalid_hint_type)
- << HintArg->getType() << HintArg->getSourceRange();
- return false;
- }
+ SemaRef.DefaultFunctionArrayLvalueConversion(TheCall->getArg(3));
+ if (HintArg.isInvalid())
+ return true;
+ auto Hint = HintArg.get();
+ if (SemaRef.convertArgumentToType(Hint, Context.IntTy))
+ return true;
+ TheCall->setArg(3, Hint);
+
+ if (!Hint->isValueDependent()) {
+ std::optional<llvm::APSInt> HintAP = Hint->getIntegerConstantExpr(Context);
+ if (!HintAP)
+ return Diag(TheCall->getBeginLoc(),
+ diag::err_atomic_hint_has_invalid_hint_type)
+ << Hint->getType() << Hint->getSourceRange();
- unsigned Hint = HintAP->getZExtValue();
- if (llvm::toAArch64MemoryHint(Hint) == llvm::AArch64MemoryHint::HINT_NONE)
+ if (llvm::toAArch64MemoryHint(HintAP->getZExtValue()) ==
+ llvm::AArch64MemoryHint::HINT_NONE) {
Diag(TheCall->getBeginLoc(), diag::warn_atomic_hint_has_invalid_hint_type)
- << *HintAP << HintArg->getSourceRange();
+ << *HintAP << Hint->getSourceRange();
+ return false;
+ }
}
-
return false;
}
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index e4cb8ba0885d0..b2ffec93bc019 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -201,14 +201,14 @@ static bool checkBuiltinVerboseTrap(CallExpr *Call, Sema &S) {
return !HasError;
}
-static bool convertArgumentToType(Sema &S, Expr *&Value, QualType Ty) {
+bool Sema::convertArgumentToType(Expr *&Value, QualType Ty) {
if (Value->isTypeDependent())
return false;
InitializedEntity Entity =
- InitializedEntity::InitializeParameter(S.Context, Ty, false);
+ InitializedEntity::InitializeParameter(Context, Ty, false);
ExprResult Result =
- S.PerformCopyInitialization(Entity, SourceLocation(), Value);
+ PerformCopyInitialization(Entity, SourceLocation(), Value);
if (Result.isInvalid())
return true;
Value = Result.get();
@@ -1702,7 +1702,7 @@ static bool checkPointerAuthEnabled(Sema &S, Expr *E) {
static bool checkPointerAuthKey(Sema &S, Expr *&Arg) {
// Convert it to type 'int'.
- if (convertArgumentToType(S, Arg, S.Context.IntTy))
+ if (S.convertArgumentToType(Arg, S.Context.IntTy))
return true;
// Value-dependent expressions are okay; wait for template instantiation.
@@ -1838,7 +1838,7 @@ static bool checkPointerAuthValue(Sema &S, Expr *&Arg, PointerAuthOpKind OpKind,
// Convert to that type. This should just be an lvalue-to-rvalue
// conversion.
- if (convertArgumentToType(S, Arg, ExpectedTy))
+ if (S.convertArgumentToType(Arg, ExpectedTy))
return true;
if (!RequireConstant) {
@@ -6825,7 +6825,7 @@ bool Sema::BuiltinAssumeAligned(CallExpr *TheCall) {
if (NumArgs > 2) {
Expr *ThirdArg = TheCall->getArg(2);
- if (convertArgumentToType(*this, ThirdArg, Context.getSizeType()))
+ if (convertArgumentToType(ThirdArg, Context.getSizeType()))
return true;
TheCall->setArg(2, ThirdArg);
}
diff --git a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
index a5f3d416f61ad..ecf1e10a09b5d 100644
--- a/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
+++ b/clang/test/CodeGen/AArch64/atomic-store-hint-template.cpp
@@ -12,24 +12,49 @@ void test_atomic_store_hint_template_order(int *ptr, int val) {
// CHECK-NEXT: [[ENTRY:.*:]]
// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] release, align 4, !mem.cache_hint [[META4:![0-9]+]]
// CHECK-NEXT: ret void
-//
__builtin_arm_atomic_store_with_hint(ptr, val, Order, 0);
}
template void test_atomic_store_hint_template_order<__ATOMIC_RELEASE>(int *, int);
template <unsigned Hint>
+void test_atomic_store_hint_template_hint(int *ptr, int val) {
// CHECK-LABEL: define weak_odr void @_Z36test_atomic_store_hint_template_hintILj1EEvPii(
// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VAL:%.*]]) #[[ATTR0]] comdat {
// CHECK-NEXT: [[ENTRY:.*:]]
// CHECK-NEXT: store atomic i32 [[VAL]], ptr [[PTR]] seq_cst, align 4, !mem.cache_hint [[META6:![0-9]+]]
// CHECK-NEXT: ret void
-void test_atomic_store_hint_template_hint(int *ptr, int val) {
__builtin_arm_atomic_store_with_hint(ptr, val, __ATOMIC_SEQ_CST, Hint);
}
template void test_atomic_store_hint_template_hint<HINT_STSHH_STRM>(int *, int);
+struct MemVal {
+ constexpr operator int() const { return __ATOMIC_RELAXED; }
+};
+
+void test_atomic_store_hint_array_consexpr_order(int *ptr, int value) {
+// CHECK-LABEL: define dso_local void @_Z43test_atomic_store_hint_array_consexpr_orderPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META4]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, value, MemVal{}, 0);
+}
+
+struct HintVal {
+ constexpr operator int() const { return 1; }
+};
+
+void test_atomic_store_hint_array_consexpr_hint(int *ptr, int value) {
+// CHECK-LABEL: define dso_local void @_Z42test_atomic_store_hint_array_consexpr_hintPii(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef [[VALUE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: store atomic i32 [[VALUE]], ptr [[PTR]] monotonic, align 4, !mem.cache_hint [[META6]]
+// CHECK-NEXT: ret void
+ __builtin_arm_atomic_store_with_hint(ptr, value, __ATOMIC_RELAXED, HintVal{});
+}
+
//.
// CHECK: [[META4]] = !{i32 1, [[META5:![0-9]+]]}
// CHECK: [[META5]] = !{!"aarch64.mem_hint", i32 0}
diff --git a/clang/test/Sema/builtins-arm64.c b/clang/test/Sema/builtins-arm64.c
index 5a14deb295a78..9a6e0715c4c78 100644
--- a/clang/test/Sema/builtins-arm64.c
+++ b/clang/test/Sema/builtins-arm64.c
@@ -75,5 +75,7 @@ void test_atomic_store_hint(char *c_ptr, __int128 *inv_ptr, float *f_ptr,
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, 3); // expected-warning {{unrecognised hint type argument to atomic hint operation (3)}}
__builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, inv_int); // expected-error {{invalid hint type argument to atomic hint operation ('int')}}
- __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{invalid hint type argument to atomic hint operation ('char *')}}
+
+ __builtin_arm_atomic_store_with_hint(c_ptr, c_data, 0, "h"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}}
+ // expected-error at -1 {{invalid hint type argument to atomic hint operation ('int')}}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b4cd198683fa0..7d185031d7d37 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5454,10 +5454,10 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
report_fatal_error("Cannot generate unaligned atomic store");
auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
- const MDNode *MemCacheHint =
- getMemCacheHintMetadata(I, I.getPointerOperandIndex());
MachineFunction &MF = DAG.getMachineFunction();
+ const MDNode *MemCacheHint =
+ getMemCacheHintMetadata(I, I.getPointerOperandIndex());
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
I.getAlign(), MMOMetadata(AAMDNodes(), /*Ranges=*/nullptr, MemCacheHint),
>From 69910ee777527c1061ed0014622e30d6c16ec1cd Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 24 Sep 2026 09:00:20 +0000
Subject: [PATCH 13/13] - Update new call to convertArgumentToType after rebase
---
clang/lib/Sema/SemaChecking.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index b2ffec93bc019..687af75c1496b 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -6710,7 +6710,7 @@ bool Sema::BuiltinPrefetch(CallExpr *TheCall) {
// Argument 0 is checked for us and the remaining arguments must be
// constant integers.
for (unsigned i = 1; i != NumArgs; ++i) {
- if (convertArgumentToType(*this, TheCall->getArgs()[i], Context.IntTy))
+ if (convertArgumentToType(TheCall->getArgs()[i], Context.IntTy))
return true;
if (BuiltinConstantArgRange(TheCall, i, 0, i == 1 ? 1 : 3))
return true;
More information about the llvm-commits
mailing list