[llvm-branch-commits] [llvm] [AArch64] Support FP atomic stores with memory hints (PR #231495)
Vimal Patel via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Oct 14 02:20:27 PDT 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/231495
>From 1e7631b1edcb7ac202bbaa501b234d4c812eddc5 Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Tue, 13 Oct 2026 10:54:21 +0000
Subject: [PATCH 1/2] [AArch64] Support direct RCPC3 FP atomic loads and stores
Preserve half, bfloat, float, and double atomic loads and stores
through AtomicExpand for acquire and release orderings, and preserve
sequentially consistent stores, when RCPC3 and NEON are available.
Add SelectionDAG patterns for the RCPC3 LDAPUR and STLUR operations,
including unscaled offsets, and update the existing FP lane patterns to
match the preserved floating-point atomic nodes. Targets without the
required features continue to use integer lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 9 +
.../lib/Target/AArch64/AArch64InstrAtomics.td | 73 ++++++-
llvm/test/CodeGen/AArch64/atomic-fp-cast.ll | 52 ++++-
llvm/test/CodeGen/AArch64/atomic-fp-load.ll | 118 ++++++++++++
llvm/test/CodeGen/AArch64/atomic-fp-store.ll | 182 ++++++++++++++++++
llvm/test/CodeGen/AArch64/rcpc3.ll | 32 +--
6 files changed, 441 insertions(+), 25 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/atomic-fp-load.ll
create mode 100644 llvm/test/CodeGen/AArch64/atomic-fp-store.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f95b129909091..d8d1409c54a6a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33234,6 +33234,10 @@ AArch64TargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
case AtomicOrdering::Unordered:
case AtomicOrdering::Monotonic:
return AtomicExpansionKind::None;
+ case AtomicOrdering::Acquire:
+ if (Subtarget->hasRCPC3() && Subtarget->isNeonAvailable())
+ return AtomicExpansionKind::None;
+ break;
default:
break;
}
@@ -33255,6 +33259,11 @@ AArch64TargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
case AtomicOrdering::Unordered:
case AtomicOrdering::Monotonic:
return AtomicExpansionKind::None;
+ case AtomicOrdering::Release:
+ case AtomicOrdering::SequentiallyConsistent:
+ if (Subtarget->hasRCPC3() && Subtarget->isNeonAvailable())
+ return AtomicExpansionKind::None;
+ break;
default:
break;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index e452467dc0656..4e34d740d30c9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -809,33 +809,90 @@ let Predicates = [HasLSFE] in {
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
+ def : Pat<(bf16 (acquiring_load<atomic_load_nonext_bf16> GPR64sp:$Rn)),
+ (LDAPURhi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(bf16 (acquiring_load<atomic_load_nonext_bf16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURhi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f16 (acquiring_load<atomic_load_nonext_f16> GPR64sp:$Rn)),
+ (LDAPURhi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f16 (acquiring_load<atomic_load_nonext_f16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURhi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f32 (acquiring_load<atomic_load_nonext_f32> GPR64sp:$Rn)),
+ (LDAPURsi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f32 (acquiring_load<atomic_load_nonext_f32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURsi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)),
+ (LDAPURdi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f64 (acquiring_load<atomic_load_nonext_f64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURdi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(releasing_store<atomic_store_bf16> GPR64sp:$Rn,
+ (bf16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_bf16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset),
+ (bf16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f16> GPR64sp:$Rn,
+ (f16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset),
+ (f16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f32> GPR64sp:$Rn,
+ (f32 FPR32:$val)),
+ (STLURsi FPR32:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset),
+ (f32 FPR32:$val)),
+ (STLURsi FPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 FPR64:$val)),
+ (STLURdi FPR64:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset),
+ (f64 FPR64:$val)),
+ (STLURdi FPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
+
// LDAP1 loads
def : Pat<(vector_insert (v2i64 VecListOne128:$Rd),
(i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)), (i64 VectorIndexD:$idx)),
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
def : Pat<(vector_insert (v2f64 VecListOne128:$Rd),
- (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)),
+ (f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)),
+ (i64 VectorIndexD:$idx)),
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
def : Pat<(v1i64 (scalar_to_vector_v1f64
(i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))),
(EXTRACT_SUBREG (LDAP1 (v2i64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;
def : Pat<(v1f64 (scalar_to_vector_v1f64
- (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))))),
- (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;
+ (f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)))),
+ (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0),
+ GPR64sp:$Rn), dsub)>;
// STL1 stores
def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
(i64 (vector_extract (v2i64 VecListOne128:$Vt), VectorIndexD:$idx))),
(STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;
- def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
- (i64 (bitconvert (f64 (vector_extract (v2f64 VecListOne128:$Vt), VectorIndexD:$idx))))),
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 (vector_extract (v2f64 VecListOne128:$Vt),
+ VectorIndexD:$idx))),
(STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;
// The v1i64 version of the vldap1_lane_* intrinsic is represented as a
// vector_insert -> vector_extract -> atomic store sequence, which is captured
// by the patterns above. We only need to cover the v1f64 case manually.
- def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
- (i64 (bitconvert (v1f64 VecListOne64:$Vt)))),
- (STL1 (SUBREG_TO_REG VecListOne64:$Vt, dsub), (i64 0), GPR64sp:$Rn)>;
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 (bitconvert (v1f64 VecListOne64:$Vt)))),
+ (STL1 (SUBREG_TO_REG VecListOne64:$Vt, dsub), (i64 0),
+ GPR64sp:$Rn)>;
}
// v8.4a FEAT_LRCPC2 patterns
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll b/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
index ef35644573e9a..4a0f6f3ec154f 100644
--- a/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
@@ -1,6 +1,9 @@
; RUN: opt -mtriple=aarch64 -S \
; RUN: -passes='require<libcall-lowering-info>,atomic-expand' %s | \
-; RUN: FileCheck %s --check-prefixes=FP,ALL
+; RUN: FileCheck %s --check-prefixes=FP,NO-RCPC3,ALL
+; RUN: opt -mtriple=aarch64 -mattr=+rcpc3 -S \
+; RUN: -passes='require<libcall-lowering-info>,atomic-expand' %s | \
+; RUN: FileCheck %s --check-prefixes=FP,RCPC3,ALL
; RUN: opt -mtriple=aarch64 -mattr=-fp-armv8 -S \
; RUN: -passes='require<libcall-lowering-info>,atomic-expand' %s | \
; RUN: FileCheck %s --check-prefixes=NOFP,ALL
@@ -29,6 +32,22 @@ define bfloat @load_monotonic_bfloat(ptr %ptr) {
ret bfloat %val
}
+define float @load_acquire_float(ptr %ptr) {
+; NO-RCPC3-LABEL: define float @load_acquire_float(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = load atomic i32, ptr [[PTR:%.*]] acquire, align 4
+; NO-RCPC3-NEXT: [[VAL:%.*]] = bitcast i32 [[INTVAL]] to float
+; NO-RCPC3-NEXT: ret float [[VAL]]
+; RCPC3-LABEL: define float @load_acquire_float(
+; RCPC3-NEXT: [[VAL:%.*]] = load atomic float, ptr [[PTR:%.*]] acquire, align 4
+; RCPC3-NEXT: ret float [[VAL]]
+; NOFP-LABEL: define float @load_acquire_float(
+; NOFP-NEXT: [[INTVAL:%.*]] = load atomic i32, ptr [[PTR:%.*]] acquire, align 4
+; NOFP-NEXT: [[VAL:%.*]] = bitcast i32 [[INTVAL]] to float
+; NOFP-NEXT: ret float [[VAL]]
+ %val = load atomic float, ptr %ptr acquire, align 4
+ ret float %val
+}
+
define double @load_seq_cst_double(ptr %ptr) {
; ALL-LABEL: define double @load_seq_cst_double(
; ALL-NEXT: [[INTVAL:%.*]] = load atomic i64, ptr [[PTR:%.*]] seq_cst, align 8
@@ -62,11 +81,34 @@ define void @store_monotonic_bfloat(ptr %ptr, bfloat %val) {
ret void
}
+define void @store_release_float(ptr %ptr, float %val) {
+; NO-RCPC3-LABEL: define void @store_release_float(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = bitcast float [[VAL:%.*]] to i32
+; NO-RCPC3-NEXT: store atomic i32 [[INTVAL]], ptr [[PTR:%.*]] release, align 4
+; NO-RCPC3-NEXT: ret void
+; RCPC3-LABEL: define void @store_release_float(
+; RCPC3-NEXT: store atomic float [[VAL:%.*]], ptr [[PTR:%.*]] release, align 4
+; RCPC3-NEXT: ret void
+; NOFP-LABEL: define void @store_release_float(
+; NOFP-NEXT: [[INTVAL:%.*]] = bitcast float [[VAL:%.*]] to i32
+; NOFP-NEXT: store atomic i32 [[INTVAL]], ptr [[PTR:%.*]] release, align 4
+; NOFP-NEXT: ret void
+ store atomic float %val, ptr %ptr release, align 4
+ ret void
+}
+
define void @store_seq_cst_double(ptr %ptr, double %val) {
-; ALL-LABEL: define void @store_seq_cst_double(
-; ALL-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
-; ALL-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
-; ALL-NEXT: ret void
+; NO-RCPC3-LABEL: define void @store_seq_cst_double(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
+; NO-RCPC3-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
+; NO-RCPC3-NEXT: ret void
+; RCPC3-LABEL: define void @store_seq_cst_double(
+; RCPC3-NEXT: store atomic double [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 8
+; RCPC3-NEXT: ret void
+; NOFP-LABEL: define void @store_seq_cst_double(
+; NOFP-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
+; NOFP-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
+; NOFP-NEXT: ret void
store atomic double %val, ptr %ptr seq_cst, align 8
ret void
}
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-load.ll b/llvm/test/CodeGen/AArch64/atomic-fp-load.ll
new file mode 100644
index 0000000000000..1e6e9653b4842
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-load.ll
@@ -0,0 +1,118 @@
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NO-RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=+rcpc3 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NOFP
+
+define half @load_acquire_half(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_half:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldarh w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3: ret
+; RCPC3-LABEL: load_acquire_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldarh w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic half, ptr %ptr acquire, align 2
+ ret half %val
+}
+
+define half @load_acquire_half_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_half_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ %val = load atomic half, ptr %addr acquire, align 2
+ ret half %val
+}
+
+define bfloat @load_acquire_bfloat(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_bfloat:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldarh w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3: ret
+; RCPC3-LABEL: load_acquire_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldarh w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic bfloat, ptr %ptr acquire, align 2
+ ret bfloat %val
+}
+
+define bfloat @load_acquire_bfloat_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_bfloat_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ %val = load atomic bfloat, ptr %addr acquire, align 2
+ ret bfloat %val
+}
+
+define float @load_acquire_float(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_float:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldar w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3-NEXT: ret
+; RCPC3-LABEL: load_acquire_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur s0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldar w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic float, ptr %ptr acquire, align 4
+ ret float %val
+}
+
+define float @load_acquire_float_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_float_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur s0, [x0, #-4]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -4
+ %val = load atomic float, ptr %addr acquire, align 4
+ ret float %val
+}
+
+define double @load_acquire_double(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_double:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldar x8, [x0]
+; NO-RCPC3-NEXT: fmov d0, x8
+; NO-RCPC3-NEXT: ret
+; RCPC3-LABEL: load_acquire_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldar x0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic double, ptr %ptr acquire, align 8
+ ret double %val
+}
+
+define double @load_acquire_double_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_double_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0, #-8]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -8
+ %val = load atomic double, ptr %addr acquire, align 8
+ ret double %val
+}
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-store.ll b/llvm/test/CodeGen/AArch64/atomic-fp-store.ll
new file mode 100644
index 0000000000000..692c510674cdf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-store.ll
@@ -0,0 +1,182 @@
+; RUN: llc -mtriple=aarch64 -mattr=+rcpc3 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=RCPC3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NO-RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NOFP
+
+define void @store_release_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_release_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_half:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 2
+ ret void
+}
+
+define void @store_release_half_offset(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_release_half_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ store atomic half %val, ptr %addr release, align 2
+ ret void
+}
+
+define void @store_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_release_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_bfloat:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 8
+ ret void
+}
+
+define void @store_release_bfloat_offset(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_release_bfloat_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ store atomic bfloat %val, ptr %addr release, align 2
+ ret void
+}
+
+define void @store_release_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_release_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_float:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr w1, [x0]
+; NOFP-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 4
+ ret void
+}
+
+define void @store_release_float_offset(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_release_float_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0, #-4]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -4
+ store atomic float %val, ptr %addr release, align 4
+ ret void
+}
+
+define void @store_release_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_release_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_double:
+; NO-RCPC3: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr x1, [x0]
+; NOFP-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8
+ ret void
+}
+
+define void @store_release_double_offset(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_release_double_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0, #-8]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -8
+ store atomic double %val, ptr %addr release, align 8
+ ret void
+}
+
+define void @store_seq_cst_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_half:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 2
+ ret void
+}
+
+define void @store_seq_cst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_bfloat:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 2
+ ret void
+}
+
+define void @store_seq_cst_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_float:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr w1, [x0]
+; NOFP-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 4
+ ret void
+}
+
+define void @store_seq_cst_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_double:
+; NO-RCPC3: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr x1, [x0]
+; NOFP-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/rcpc3.ll b/llvm/test/CodeGen/AArch64/rcpc3.ll
index 45770337c1320..5682f8f504881 100644
--- a/llvm/test/CodeGen/AArch64/rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/rcpc3.ll
@@ -214,26 +214,34 @@ define hidden void @test_stl1_1xdouble_lane0(ptr nocapture noundef writeonly %a,
ret void
}
-; The remaining tests do not have any particular RCPC3-specific codegen:
-
; load-acquire a plain non-vector double value
define hidden double @test_double_load(ptr nocapture noundef readonly %a) local_unnamed_addr {
-; BOTH-LABEL: test_double_load:
-; BOTH: // %bb.0:
-; BOTH-NEXT: ldapr x8, [x0]
-; BOTH-NEXT: fmov d0, x8
-; BOTH-NEXT: ret
+; RCPC3-LABEL: test_double_load:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_double_load:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldapr x8, [x0]
+; NO-RCPC3-NEXT: fmov d0, x8
+; NO-RCPC3-NEXT: ret
%1 = load atomic double, ptr %a acquire, align 8
ret double %1
}
; store-release a plain non-vector double value
define hidden void @test_double_store(ptr nocapture noundef writeonly %a, double noundef %b) local_unnamed_addr {
-; BOTH-LABEL: test_double_store:
-; BOTH: // %bb.0:
-; BOTH-NEXT: fmov x8, d0
-; BOTH-NEXT: stlr x8, [x0]
-; BOTH-NEXT: ret
+; RCPC3-LABEL: test_double_store:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_double_store:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
store atomic double %b, ptr %a release, align 8
ret void
}
>From cc081b83c47ed32ddb11ae4243449c436a9db97f Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Tue, 13 Oct 2026 10:55:46 +0000
Subject: [PATCH 2/2] [AArch64] Support FP atomic stores with memory hints
This patch extends the atomic-store-hint pseudos to accept preserved
floating-point atomic stores. It adds an FPR16 pseudo family and direct
half, bfloat, float, and double patterns for the supported addressing
modes.
The pseudos expand relaxed stores to STR and RCPC3 release and
sequentially consistent stores to STLUR, retaining memory hints without
cross-register-bank copies.
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 34 +++-
.../Target/AArch64/AArch64ISelLowering.cpp | 5 -
.../lib/Target/AArch64/AArch64InstrAtomics.td | 59 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 4 +
.../Atomics/aarch64-atomic-store-hint.ll | 8 +-
.../Atomics/aarch64-relaxed-store-hint.ll | 32 +---
llvm/test/CodeGen/AArch64/rcpc3.ll | 145 ++++++++++++++++++
7 files changed, 251 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c3755d713f8af..36cb4142ce8d9 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3252,6 +3252,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
unsigned StOpc;
unsigned Relaxed = MI->getOperand(2).getImm();
assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
+ bool EmitZeroImm = Relaxed;
switch (MI->getOpcode()) {
case AArch64::ATOMIC_STORE_HINT_B:
StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
@@ -3265,6 +3266,18 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_X:
StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
break;
+ case AArch64::ATOMIC_STORE_HINT_FPR16:
+ StOpc = Relaxed ? AArch64::STRHui : AArch64::STLURhi;
+ EmitZeroImm = true;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_S:
+ StOpc = Relaxed ? AArch64::STRSui : AArch64::STLURsi;
+ EmitZeroImm = true;
+ break;
+ case AArch64::ATOMIC_STORE_HINT_D:
+ StOpc = Relaxed ? AArch64::STRDui : AArch64::STLURdi;
+ EmitZeroImm = true;
+ break;
default:
llvm_unreachable("Unexpected atomic hint size.");
}
@@ -3277,7 +3290,7 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
Store.setFlags(MI->getFlags());
- if (Relaxed)
+ if (EmitZeroImm)
Store.addOperand(MCOperand::createImm(0));
EmitToStreamer(*OutStreamer, Store);
}
@@ -3301,6 +3314,9 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
case AArch64::ATOMIC_STORE_HINT_XroW:
StOpc = AArch64::STRXroW;
break;
+ case AArch64::ATOMIC_STORE_HINT_FPR16roW:
+ StOpc = AArch64::STRHroW;
+ break;
case AArch64::ATOMIC_STORE_HINT_SroW:
StOpc = AArch64::STRSroW;
break;
@@ -3319,6 +3335,9 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
case AArch64::ATOMIC_STORE_HINT_XroX:
StOpc = AArch64::STRXroX;
break;
+ case AArch64::ATOMIC_STORE_HINT_FPR16roX:
+ StOpc = AArch64::STRHroX;
+ break;
case AArch64::ATOMIC_STORE_HINT_SroX:
StOpc = AArch64::STRSroX;
break;
@@ -3362,6 +3381,9 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
case AArch64::ATOMIC_STORE_HINT_Xui:
StOpc = AArch64::STRXui;
break;
+ case AArch64::ATOMIC_STORE_HINT_FPR16ui:
+ StOpc = AArch64::STRHui;
+ break;
case AArch64::ATOMIC_STORE_HINT_Sui:
StOpc = AArch64::STRSui;
break;
@@ -3380,6 +3402,9 @@ void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
case AArch64::ATOMIC_STORE_HINT_Xi:
StOpc = AArch64::STURXi;
break;
+ case AArch64::ATOMIC_STORE_HINT_FPR16i:
+ StOpc = AArch64::STURHi;
+ break;
case AArch64::ATOMIC_STORE_HINT_Si:
StOpc = AArch64::STURSi;
break;
@@ -4123,18 +4148,23 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_H:
case AArch64::ATOMIC_STORE_HINT_W:
case AArch64::ATOMIC_STORE_HINT_X:
+ case AArch64::ATOMIC_STORE_HINT_FPR16:
+ case AArch64::ATOMIC_STORE_HINT_S:
+ case AArch64::ATOMIC_STORE_HINT_D:
emitAtomicHintPseudoExpansion(MI);
return;
case AArch64::ATOMIC_STORE_HINT_BroW:
case AArch64::ATOMIC_STORE_HINT_HroW:
case AArch64::ATOMIC_STORE_HINT_WroW:
case AArch64::ATOMIC_STORE_HINT_XroW:
+ case AArch64::ATOMIC_STORE_HINT_FPR16roW:
case AArch64::ATOMIC_STORE_HINT_SroW:
case AArch64::ATOMIC_STORE_HINT_DroW:
case AArch64::ATOMIC_STORE_HINT_BroX:
case AArch64::ATOMIC_STORE_HINT_HroX:
case AArch64::ATOMIC_STORE_HINT_WroX:
case AArch64::ATOMIC_STORE_HINT_XroX:
+ case AArch64::ATOMIC_STORE_HINT_FPR16roX:
case AArch64::ATOMIC_STORE_HINT_SroX:
case AArch64::ATOMIC_STORE_HINT_DroX:
emitAtomicHintPseudoExpansionRO(MI);
@@ -4143,12 +4173,14 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
case AArch64::ATOMIC_STORE_HINT_Hui:
case AArch64::ATOMIC_STORE_HINT_Wui:
case AArch64::ATOMIC_STORE_HINT_Xui:
+ case AArch64::ATOMIC_STORE_HINT_FPR16ui:
case AArch64::ATOMIC_STORE_HINT_Sui:
case AArch64::ATOMIC_STORE_HINT_Dui:
case AArch64::ATOMIC_STORE_HINT_Bi:
case AArch64::ATOMIC_STORE_HINT_Hi:
case AArch64::ATOMIC_STORE_HINT_Wi:
case AArch64::ATOMIC_STORE_HINT_Xi:
+ case AArch64::ATOMIC_STORE_HINT_FPR16i:
case AArch64::ATOMIC_STORE_HINT_Si:
case AArch64::ATOMIC_STORE_HINT_Di:
emitAtomicHintPseudoExpansionImm(MI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d8d1409c54a6a..6335bc31ad2d5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33248,11 +33248,6 @@ AArch64TargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
TargetLoweringBase::AtomicExpansionKind
AArch64TargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
- // Atomic stores with memory hints use target pseudos that currently expect
- // non-integral values to be cast to integers.
- if (SI->getMetadata(LLVMContext::MD_mem_cache_hint))
- return TargetLowering::shouldCastAtomicStoreInIR(SI);
-
if (Subtarget->hasFPARMv8() &&
isSupportedFPAtomicType(SI->getValueOperand()->getType())) {
switch (SI->getOrdering()) {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 4e34d740d30c9..8406265a0461a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -403,6 +403,7 @@ defm ATOMIC_STORE_HINT_B : AtomicStoreHintPseudos<GPR32, ro_Wextend8, ro_Xexten
defm ATOMIC_STORE_HINT_H : AtomicStoreHintPseudos<GPR32, ro_Wextend16, ro_Xextend16, uimm12s2>;
defm ATOMIC_STORE_HINT_W : AtomicStoreHintPseudos<GPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
defm ATOMIC_STORE_HINT_X : AtomicStoreHintPseudos<GPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
+defm ATOMIC_STORE_HINT_FPR16 : AtomicStoreHintPseudos<FPR16, ro_Wextend16, ro_Xextend16, uimm12s2>;
defm ATOMIC_STORE_HINT_S : AtomicStoreHintPseudos<FPR32, ro_Wextend32, ro_Xextend32, uimm12s4>;
defm ATOMIC_STORE_HINT_D : AtomicStoreHintPseudos<FPR64, ro_Wextend64, ro_Xextend64, uimm12s8>;
@@ -432,6 +433,18 @@ multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
(ATOMIC_STORE_HINT_DroW FPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f16, 1, Pred>
+ (f16 FPR16Op:$data), (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
+ (ATOMIC_STORE_HINT_FPR16roW FPR16:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_bf16, 1, Pred>
+ (bf16 FPR16Op:$data), (ro_Windexed16 GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend)),
+ (ATOMIC_STORE_HINT_FPR16roW FPR16:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f32, 1, Pred>
+ (f32 FPR32Op:$data), (ro_Windexed32 GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroW FPR32:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f64, 1, Pred>
+ (f64 FPR64Op:$data), (ro_Windexed64 GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroW FPR64:$data, GPR64sp:$addr, GPR32:$Rm, ro_Wextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (ro_Xindexed8 GPR64sp:$addr, GPR64:$Rm, ro_Xextend8:$extend)),
@@ -451,6 +464,18 @@ multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
(ATOMIC_STORE_HINT_DroX FPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f16, 1, Pred>
+ (f16 FPR16Op:$data), (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
+ (ATOMIC_STORE_HINT_FPR16roX FPR16:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_bf16, 1, Pred>
+ (bf16 FPR16Op:$data), (ro_Xindexed16 GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend)),
+ (ATOMIC_STORE_HINT_FPR16roX FPR16:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend16:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f32, 1, Pred>
+ (f32 FPR32Op:$data), (ro_Xindexed32 GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend)),
+ (ATOMIC_STORE_HINT_SroX FPR32:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend32:$extend, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f64, 1, Pred>
+ (f64 FPR64Op:$data), (ro_Xindexed64 GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend)),
+ (ATOMIC_STORE_HINT_DroX FPR64:$data, GPR64sp:$addr, GPR64:$Rm, ro_Xextend64:$extend, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_indexed8 GPR64sp:$addr, uimm12s1:$offset)),
@@ -470,6 +495,18 @@ multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
(ATOMIC_STORE_HINT_Dui FPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f16, 1, Pred>
+ (f16 FPR16Op:$data), (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
+ (ATOMIC_STORE_HINT_FPR16ui FPR16:$data, GPR64sp:$addr, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_bf16, 1, Pred>
+ (bf16 FPR16Op:$data), (am_indexed16 GPR64sp:$addr, uimm12s2:$offset)),
+ (ATOMIC_STORE_HINT_FPR16ui FPR16:$data, GPR64sp:$addr, uimm12s2:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f32, 1, Pred>
+ (f32 FPR32Op:$data), (am_indexed32 GPR64sp:$addr, uimm12s4:$offset)),
+ (ATOMIC_STORE_HINT_Sui FPR32:$data, GPR64sp:$addr, uimm12s4:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f64, 1, Pred>
+ (f64 FPR64Op:$data), (am_indexed64 GPR64sp:$addr, uimm12s8:$offset)),
+ (ATOMIC_STORE_HINT_Dui FPR64:$data, GPR64sp:$addr, uimm12s8:$offset, (i32 1), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_8, 1, Pred>
GPR32:$data, (am_unscaled8 GPR64sp:$addr, simm9:$offset)),
@@ -489,6 +526,18 @@ multiclass AtomicHintPatternsAddrMode<int Hint, code Pred> {
def : Pat<(atomic_hint_store<atomic_store_64, 1, Pred>
(i64 (bitconvert (f64 FPR64Op:$data))), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
(ATOMIC_STORE_HINT_Di FPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f16, 1, Pred>
+ (f16 FPR16Op:$data), (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_FPR16i FPR16:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_bf16, 1, Pred>
+ (bf16 FPR16Op:$data), (am_unscaled16 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_FPR16i FPR16:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f32, 1, Pred>
+ (f32 FPR32Op:$data), (am_unscaled32 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Si FPR32:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f64, 1, Pred>
+ (f64 FPR64Op:$data), (am_unscaled64 GPR64sp:$addr, simm9:$offset)),
+ (ATOMIC_STORE_HINT_Di FPR64:$data, GPR64sp:$addr, simm9:$offset, (i32 1), (i32 Hint))>;
}
multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
@@ -500,6 +549,16 @@ multiclass AtomicHintPatterns<int Relaxed, int Hint, code Pred> {
(ATOMIC_STORE_HINT_W GPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
def : Pat<(atomic_hint_store<atomic_store_64, Relaxed, Pred> GPR64:$data, GPR64sp:$addr),
(ATOMIC_STORE_HINT_X GPR64:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ let Predicates = [HasRCPC3, HasNEON] in {
+ def : Pat<(atomic_hint_store<atomic_store_f16, Relaxed, Pred> (f16 FPR16Op:$data), GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_FPR16 FPR16:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_bf16, Relaxed, Pred> (bf16 FPR16Op:$data), GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_FPR16 FPR16:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f32, Relaxed, Pred> (f32 FPR32Op:$data), GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_S FPR32:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ def : Pat<(atomic_hint_store<atomic_store_f64, Relaxed, Pred> (f64 FPR64Op:$data), GPR64sp:$addr),
+ (ATOMIC_STORE_HINT_D FPR64:$data, GPR64sp:$addr, (i32 Relaxed), (i32 Hint))>;
+ }
}
let AddedComplexity = 1, GISelShouldIgnore = 1 in {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 8b0552623ad81..2e83402a2e2a4 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -3124,12 +3124,14 @@ unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
case AArch64::ATOMIC_STORE_HINT_Bi:
case AArch64::ATOMIC_STORE_HINT_Hi:
case AArch64::ATOMIC_STORE_HINT_Wi:
+ case AArch64::ATOMIC_STORE_HINT_FPR16i:
case AArch64::ATOMIC_STORE_HINT_Si:
case AArch64::ATOMIC_STORE_HINT_Xi:
case AArch64::ATOMIC_STORE_HINT_Di:
case AArch64::ATOMIC_STORE_HINT_Bui:
case AArch64::ATOMIC_STORE_HINT_Hui:
case AArch64::ATOMIC_STORE_HINT_Wui:
+ case AArch64::ATOMIC_STORE_HINT_FPR16ui:
case AArch64::ATOMIC_STORE_HINT_Sui:
case AArch64::ATOMIC_STORE_HINT_Xui:
case AArch64::ATOMIC_STORE_HINT_Dui:
@@ -4764,6 +4766,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STRHui:
case AArch64::STRHHui:
case AArch64::ATOMIC_STORE_HINT_Hui:
+ case AArch64::ATOMIC_STORE_HINT_FPR16ui:
Scale = Width = TypeSize::getFixed(2);
MinOffset = 0;
MaxOffset = 4095;
@@ -4886,6 +4889,7 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
case AArch64::STURHHi:
case AArch64::STLURHi:
case AArch64::ATOMIC_STORE_HINT_Hi:
+ case AArch64::ATOMIC_STORE_HINT_FPR16i:
Scale = TypeSize::getFixed(1);
Width = TypeSize::getFixed(2);
MinOffset = -256;
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
index 2528a4079b032..1b2268a27aaa4 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomic-store-hint.ll
@@ -171,10 +171,8 @@ define void @test_atomic_store_strm_relaxed_mfloat(ptr %ptr, <1 x i8> %val) noun
define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_bfloat:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: str h0, [x0]
; CHECK-NEXT: ret
store atomic bfloat %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
@@ -183,10 +181,8 @@ define void @test_atomic_store_strm_relaxed_bfloat(ptr %ptr, bfloat %val) nounwi
define void @test_atomic_store_strm_relaxed_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: test_atomic_store_strm_relaxed_half:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: strh w8, [x0]
+; CHECK-NEXT: str h0, [x0]
; CHECK-NEXT: ret
store atomic half %val, ptr %ptr monotonic, align 8, !mem.cache_hint !2
ret void
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
index 74d50efa988bd..5ce8723215d3b 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-relaxed-store-hint.ll
@@ -70,10 +70,8 @@ define void @relaxed_store_hint_roW_mfloat(ptr %ptr, i32 %offset, <1 x i8> %val)
define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_bfloat:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: str h0, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 %offset
store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -83,10 +81,8 @@ define void @relaxed_store_hint_roW_bfloat(ptr %ptr, i32 %offset, bfloat %val) n
define void @relaxed_store_hint_roW_half(ptr %ptr, i32 %offset, half %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roW_half:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: strh w8, [x0, w1, sxtw #1]
+; CHECK-NEXT: str h0, [x0, w1, sxtw #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 %offset
store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -183,10 +179,8 @@ define void @relaxed_store_hint_roX_mfloat(ptr %ptr, i64 %offset, <1 x i8> %val)
define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_bfloat:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: str h0, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i64 %offset
store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -196,10 +190,8 @@ define void @relaxed_store_hint_roX_bfloat(ptr %ptr, i64 %offset, bfloat %val) n
define void @relaxed_store_hint_roX_half(ptr %ptr, i64 %offset, half %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_roX_half:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: strh w8, [x0, x1, lsl #1]
+; CHECK-NEXT: str h0, [x0, x1, lsl #1]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i64 %offset
store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -296,10 +288,8 @@ define void @relaxed_store_hint_uimm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_bfloat:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: str h0, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 2048
store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -309,10 +299,8 @@ define void @relaxed_store_hint_uimm_bfloat(ptr %ptr, bfloat %val) nounwind {
define void @relaxed_store_hint_uimm_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_uimm_half:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh keep
-; CHECK-NEXT: strh w8, [x0, #4096]
+; CHECK-NEXT: str h0, [x0, #4096]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 2048
store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !0
@@ -409,10 +397,8 @@ define void @relaxed_store_hint_imm_mfloat(ptr %ptr, <1 x i8> %val) nounwind {
define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_bfloat:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: stur h0, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr bfloat, ptr %ptr, i32 -128
store atomic bfloat %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
@@ -422,10 +408,8 @@ define void @relaxed_store_hint_imm_bfloat(ptr %ptr, bfloat %val) nounwind {
define void @relaxed_store_hint_imm_half(ptr %ptr, half %val) nounwind {
; CHECK-LABEL: relaxed_store_hint_imm_half:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $h0 killed $h0 def $s0
-; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: stshh strm
-; CHECK-NEXT: sturh w8, [x0, #-256]
+; CHECK-NEXT: stur h0, [x0, #-256]
; CHECK-NEXT: ret
%addr = getelementptr half, ptr %ptr, i32 -128
store atomic half %val, ptr %addr monotonic, align 8, !mem.cache_hint !2
diff --git a/llvm/test/CodeGen/AArch64/rcpc3.ll b/llvm/test/CodeGen/AArch64/rcpc3.ll
index 5682f8f504881..68ce29e74487e 100644
--- a/llvm/test/CodeGen/AArch64/rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/rcpc3.ll
@@ -331,3 +331,148 @@ define hidden void @test_extract_store_2xi64(ptr nocapture noundef writeonly %a,
store i64 %1, ptr %a, align 8
ret void
}
+
+define void @test_atomic_store_hint_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_release_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh keep
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_release_bfloat:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: // kill: def $h0 killed $h0 def $s0
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh keep
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 2, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_hint_release_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_release_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh keep
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_release_half:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: // kill: def $h0 killed $h0 def $s0
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh keep
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 2, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_hint_release_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_release_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh keep
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_release_float:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh keep
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 4, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_hint_release_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_release_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh keep
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_release_double:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov x8, d0
+; NO-RCPC3-NEXT: stshh keep
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8, !mem.cache_hint !0
+ ret void
+}
+
+define void @test_atomic_store_hint_seq_cst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_seq_cst_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh strm
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_seq_cst_bfloat:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: // kill: def $h0 killed $h0 def $s0
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh strm
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 2, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_hint_seq_cst_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_seq_cst_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh strm
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_seq_cst_half:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: // kill: def $h0 killed $h0 def $s0
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh strm
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 2, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_hint_seq_cst_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_seq_cst_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh strm
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_seq_cst_float:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov w8, s0
+; NO-RCPC3-NEXT: stshh strm
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 4, !mem.cache_hint !2
+ ret void
+}
+
+define void @test_atomic_store_hint_seq_cst_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: test_atomic_store_hint_seq_cst_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stshh strm
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_atomic_store_hint_seq_cst_double:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov x8, d0
+; NO-RCPC3-NEXT: stshh strm
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8, !mem.cache_hint !2
+ ret void
+}
+
+!0 = !{i32 1, !1}
+!1 = !{!"aarch64.mem_hint", i32 0}
+!2 = !{i32 1, !3}
+!3 = !{!"aarch64.mem_hint", i32 1}
More information about the llvm-branch-commits
mailing list