[llvm-branch-commits] [llvm] [AArch64] Support direct RCPC3 FP atomic loads and stores (PR #231492)
Vimal Patel via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Oct 20 08:01:50 PDT 2026
https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/231492
>From d491a3623afe7ca8862820fc87f7a7a3eacd379c Mon Sep 17 00:00:00 2001
From: Vimal Patel <vimal.patel at arm.com>
Date: Tue, 13 Oct 2026 10:54:21 +0000
Subject: [PATCH] [AArch64] Support direct RCPC3 FP atomic loads and stores
Preserve half, bfloat, float, and double atomic loads and stores
through AtomicExpand for acquire and release orderings, and preserve
sequentially consistent stores, when RCPC3 and NEON are available.
Add SelectionDAG patterns for the RCPC3 LDAPUR and STLUR operations,
including unscaled offsets, and update the existing FP lane patterns to
match the preserved floating-point atomic nodes. Targets without the
required features continue to use integer lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 9 +
.../lib/Target/AArch64/AArch64InstrAtomics.td | 73 ++++++-
llvm/test/CodeGen/AArch64/atomic-fp-cast.ll | 50 ++++-
llvm/test/CodeGen/AArch64/atomic-fp-load.ll | 118 ++++++++++++
llvm/test/CodeGen/AArch64/atomic-fp-store.ll | 182 ++++++++++++++++++
llvm/test/CodeGen/AArch64/rcpc3.ll | 32 +--
6 files changed, 439 insertions(+), 25 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/atomic-fp-load.ll
create mode 100644 llvm/test/CodeGen/AArch64/atomic-fp-store.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f95b1299090911..d8d1409c54a6a4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33234,6 +33234,10 @@ AArch64TargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
case AtomicOrdering::Unordered:
case AtomicOrdering::Monotonic:
return AtomicExpansionKind::None;
+ case AtomicOrdering::Acquire:
+ if (Subtarget->hasRCPC3() && Subtarget->isNeonAvailable())
+ return AtomicExpansionKind::None;
+ break;
default:
break;
}
@@ -33255,6 +33259,11 @@ AArch64TargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
case AtomicOrdering::Unordered:
case AtomicOrdering::Monotonic:
return AtomicExpansionKind::None;
+ case AtomicOrdering::Release:
+ case AtomicOrdering::SequentiallyConsistent:
+ if (Subtarget->hasRCPC3() && Subtarget->isNeonAvailable())
+ return AtomicExpansionKind::None;
+ break;
default:
break;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index a0ce8c93b52328..417e476fd8939b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -810,33 +810,90 @@ let Predicates = [HasLSFE] in {
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
+ def : Pat<(bf16 (acquiring_load<atomic_load_nonext_bf16> GPR64sp:$Rn)),
+ (LDAPURhi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(bf16 (acquiring_load<atomic_load_nonext_bf16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURhi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f16 (acquiring_load<atomic_load_nonext_f16> GPR64sp:$Rn)),
+ (LDAPURhi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f16 (acquiring_load<atomic_load_nonext_f16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURhi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f32 (acquiring_load<atomic_load_nonext_f32> GPR64sp:$Rn)),
+ (LDAPURsi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f32 (acquiring_load<atomic_load_nonext_f32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURsi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)),
+ (LDAPURdi GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(f64 (acquiring_load<atomic_load_nonext_f64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset))),
+ (LDAPURdi GPR64sp:$Rn, simm9:$offset)>;
+
+ def : Pat<(releasing_store<atomic_store_bf16> GPR64sp:$Rn,
+ (bf16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_bf16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset),
+ (bf16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f16> GPR64sp:$Rn,
+ (f16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f16>
+ (am_unscaled16 GPR64sp:$Rn, simm9:$offset),
+ (f16 FPR16:$val)),
+ (STLURhi FPR16:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f32> GPR64sp:$Rn,
+ (f32 FPR32:$val)),
+ (STLURsi FPR32:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f32>
+ (am_unscaled32 GPR64sp:$Rn, simm9:$offset),
+ (f32 FPR32:$val)),
+ (STLURsi FPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 FPR64:$val)),
+ (STLURdi FPR64:$val, GPR64sp:$Rn, (i64 0))>;
+ def : Pat<(releasing_store<atomic_store_f64>
+ (am_unscaled64 GPR64sp:$Rn, simm9:$offset),
+ (f64 FPR64:$val)),
+ (STLURdi FPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
+
// LDAP1 loads
def : Pat<(vector_insert (v2i64 VecListOne128:$Rd),
(i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)), (i64 VectorIndexD:$idx)),
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
def : Pat<(vector_insert (v2f64 VecListOne128:$Rd),
- (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))), (i64 VectorIndexD:$idx)),
+ (f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)),
+ (i64 VectorIndexD:$idx)),
(LDAP1 VecListOne128:$Rd, VectorIndexD:$idx, GPR64sp:$Rn)>;
def : Pat<(v1i64 (scalar_to_vector_v1f64
(i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))),
(EXTRACT_SUBREG (LDAP1 (v2i64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;
def : Pat<(v1f64 (scalar_to_vector_v1f64
- (f64 (bitconvert (i64 (acquiring_load<atomic_load_nonext_64> GPR64sp:$Rn)))))),
- (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0), GPR64sp:$Rn), dsub)>;
+ (f64 (acquiring_load<atomic_load_nonext_f64> GPR64sp:$Rn)))),
+ (EXTRACT_SUBREG (LDAP1 (v2f64 (IMPLICIT_DEF)), (i64 0),
+ GPR64sp:$Rn), dsub)>;
// STL1 stores
def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
(i64 (vector_extract (v2i64 VecListOne128:$Vt), VectorIndexD:$idx))),
(STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;
- def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
- (i64 (bitconvert (f64 (vector_extract (v2f64 VecListOne128:$Vt), VectorIndexD:$idx))))),
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 (vector_extract (v2f64 VecListOne128:$Vt),
+ VectorIndexD:$idx))),
(STL1 VecListOne128:$Vt, VectorIndexD:$idx, GPR64sp:$Rn)>;
// The v1i64 version of the vldap1_lane_* intrinsic is represented as a
// vector_insert -> vector_extract -> atomic store sequence, which is captured
// by the patterns above. We only need to cover the v1f64 case manually.
- def : Pat<(releasing_store<atomic_store_64> GPR64sp:$Rn,
- (i64 (bitconvert (v1f64 VecListOne64:$Vt)))),
- (STL1 (SUBREG_TO_REG VecListOne64:$Vt, dsub), (i64 0), GPR64sp:$Rn)>;
+ def : Pat<(releasing_store<atomic_store_f64> GPR64sp:$Rn,
+ (f64 (bitconvert (v1f64 VecListOne64:$Vt)))),
+ (STL1 (SUBREG_TO_REG VecListOne64:$Vt, dsub), (i64 0),
+ GPR64sp:$Rn)>;
}
// v8.4a FEAT_LRCPC2 patterns
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll b/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
index 28d7a72d1c0f81..813985f94e24e6 100644
--- a/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-cast.ll
@@ -1,4 +1,5 @@
-; RUN: opt -mtriple=aarch64 -S -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s --check-prefixes=FP,ALL
+; RUN: opt -mtriple=aarch64 -S -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s --check-prefixes=FP,NO-RCPC3,ALL
+; RUN: opt -mtriple=aarch64 -mattr=+rcpc3 -S -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s --check-prefixes=FP,RCPC3,ALL
; RUN: opt -mtriple=aarch64 -mattr=-fp-armv8 -S -passes='require<libcall-lowering-info>,atomic-expand' %s | FileCheck %s --check-prefixes=NOFP,ALL
define half @load_unordered_half(ptr %ptr) {
@@ -25,6 +26,22 @@ define bfloat @load_monotonic_bfloat(ptr %ptr) {
ret bfloat %val
}
+define float @load_acquire_float(ptr %ptr) {
+; NO-RCPC3-LABEL: define float @load_acquire_float(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = load atomic i32, ptr [[PTR:%.*]] acquire, align 4
+; NO-RCPC3-NEXT: [[VAL:%.*]] = bitcast i32 [[INTVAL]] to float
+; NO-RCPC3-NEXT: ret float [[VAL]]
+; RCPC3-LABEL: define float @load_acquire_float(
+; RCPC3-NEXT: [[VAL:%.*]] = load atomic float, ptr [[PTR:%.*]] acquire, align 4
+; RCPC3-NEXT: ret float [[VAL]]
+; NOFP-LABEL: define float @load_acquire_float(
+; NOFP-NEXT: [[INTVAL:%.*]] = load atomic i32, ptr [[PTR:%.*]] acquire, align 4
+; NOFP-NEXT: [[VAL:%.*]] = bitcast i32 [[INTVAL]] to float
+; NOFP-NEXT: ret float [[VAL]]
+ %val = load atomic float, ptr %ptr acquire, align 4
+ ret float %val
+}
+
define double @load_seq_cst_double(ptr %ptr) {
; ALL-LABEL: define double @load_seq_cst_double(
; ALL-NEXT: [[INTVAL:%.*]] = load atomic i64, ptr [[PTR:%.*]] seq_cst, align 8
@@ -58,11 +75,34 @@ define void @store_monotonic_bfloat(ptr %ptr, bfloat %val) {
ret void
}
+define void @store_release_float(ptr %ptr, float %val) {
+; NO-RCPC3-LABEL: define void @store_release_float(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = bitcast float [[VAL:%.*]] to i32
+; NO-RCPC3-NEXT: store atomic i32 [[INTVAL]], ptr [[PTR:%.*]] release, align 4
+; NO-RCPC3-NEXT: ret void
+; RCPC3-LABEL: define void @store_release_float(
+; RCPC3-NEXT: store atomic float [[VAL:%.*]], ptr [[PTR:%.*]] release, align 4
+; RCPC3-NEXT: ret void
+; NOFP-LABEL: define void @store_release_float(
+; NOFP-NEXT: [[INTVAL:%.*]] = bitcast float [[VAL:%.*]] to i32
+; NOFP-NEXT: store atomic i32 [[INTVAL]], ptr [[PTR:%.*]] release, align 4
+; NOFP-NEXT: ret void
+ store atomic float %val, ptr %ptr release, align 4
+ ret void
+}
+
define void @store_seq_cst_double(ptr %ptr, double %val) {
-; ALL-LABEL: define void @store_seq_cst_double(
-; ALL-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
-; ALL-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
-; ALL-NEXT: ret void
+; NO-RCPC3-LABEL: define void @store_seq_cst_double(
+; NO-RCPC3-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
+; NO-RCPC3-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
+; NO-RCPC3-NEXT: ret void
+; RCPC3-LABEL: define void @store_seq_cst_double(
+; RCPC3-NEXT: store atomic double [[VAL:%.*]], ptr [[PTR:%.*]] seq_cst, align 8
+; RCPC3-NEXT: ret void
+; NOFP-LABEL: define void @store_seq_cst_double(
+; NOFP-NEXT: [[INTVAL:%.*]] = bitcast double [[VAL:%.*]] to i64
+; NOFP-NEXT: store atomic i64 [[INTVAL]], ptr [[PTR:%.*]] seq_cst, align 8
+; NOFP-NEXT: ret void
store atomic double %val, ptr %ptr seq_cst, align 8
ret void
}
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-load.ll b/llvm/test/CodeGen/AArch64/atomic-fp-load.ll
new file mode 100644
index 00000000000000..1e6e9653b4842e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-load.ll
@@ -0,0 +1,118 @@
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NO-RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=+rcpc3 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NOFP
+
+define half @load_acquire_half(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_half:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldarh w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3: ret
+; RCPC3-LABEL: load_acquire_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldarh w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic half, ptr %ptr acquire, align 2
+ ret half %val
+}
+
+define half @load_acquire_half_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_half_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ %val = load atomic half, ptr %addr acquire, align 2
+ ret half %val
+}
+
+define bfloat @load_acquire_bfloat(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_bfloat:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldarh w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3: ret
+; RCPC3-LABEL: load_acquire_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldarh w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic bfloat, ptr %ptr acquire, align 2
+ ret bfloat %val
+}
+
+define bfloat @load_acquire_bfloat_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_bfloat_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ %val = load atomic bfloat, ptr %addr acquire, align 2
+ ret bfloat %val
+}
+
+define float @load_acquire_float(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_float:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldar w8, [x0]
+; NO-RCPC3-NEXT: fmov s0, w8
+; NO-RCPC3-NEXT: ret
+; RCPC3-LABEL: load_acquire_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur s0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldar w0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic float, ptr %ptr acquire, align 4
+ ret float %val
+}
+
+define float @load_acquire_float_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_float_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur s0, [x0, #-4]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -4
+ %val = load atomic float, ptr %addr acquire, align 4
+ ret float %val
+}
+
+define double @load_acquire_double(ptr %ptr) {
+; NO-RCPC3-LABEL: load_acquire_double:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldar x8, [x0]
+; NO-RCPC3-NEXT: fmov d0, x8
+; NO-RCPC3-NEXT: ret
+; RCPC3-LABEL: load_acquire_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0]
+; RCPC3-NEXT: ret
+; NOFP-LABEL: load_acquire_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: ldar x0, [x0]
+; NOFP-NEXT: ret
+ %val = load atomic double, ptr %ptr acquire, align 8
+ ret double %val
+}
+
+define double @load_acquire_double_offset(ptr %ptr) {
+; RCPC3-LABEL: load_acquire_double_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0, #-8]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -8
+ %val = load atomic double, ptr %addr acquire, align 8
+ ret double %val
+}
diff --git a/llvm/test/CodeGen/AArch64/atomic-fp-store.ll b/llvm/test/CodeGen/AArch64/atomic-fp-store.ll
new file mode 100644
index 00000000000000..692c510674cdf4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/atomic-fp-store.ll
@@ -0,0 +1,182 @@
+; RUN: llc -mtriple=aarch64 -mattr=+rcpc3 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=RCPC3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NO-RCPC3
+; RUN: llc -mtriple=aarch64 -mattr=-fp-armv8 -verify-machineinstrs < %s | \
+; RUN: FileCheck %s --check-prefix=NOFP
+
+define void @store_release_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_release_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_half:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic half %val, ptr %ptr release, align 2
+ ret void
+}
+
+define void @store_release_half_offset(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_release_half_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ store atomic half %val, ptr %addr release, align 2
+ ret void
+}
+
+define void @store_release_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_release_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_bfloat:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic bfloat %val, ptr %ptr release, align 8
+ ret void
+}
+
+define void @store_release_bfloat_offset(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_release_bfloat_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0, #-2]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -2
+ store atomic bfloat %val, ptr %addr release, align 2
+ ret void
+}
+
+define void @store_release_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_release_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_float:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr w1, [x0]
+; NOFP-NEXT: ret
+ store atomic float %val, ptr %ptr release, align 4
+ ret void
+}
+
+define void @store_release_float_offset(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_release_float_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0, #-4]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -4
+ store atomic float %val, ptr %addr release, align 4
+ ret void
+}
+
+define void @store_release_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_release_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_release_double:
+; NO-RCPC3: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_release_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr x1, [x0]
+; NOFP-NEXT: ret
+ store atomic double %val, ptr %ptr release, align 8
+ ret void
+}
+
+define void @store_release_double_offset(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_release_double_offset:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0, #-8]
+; RCPC3-NEXT: ret
+ %addr = getelementptr i8, ptr %ptr, i64 -8
+ store atomic double %val, ptr %addr release, align 8
+ ret void
+}
+
+define void @store_seq_cst_half(ptr %ptr, half %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_half:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_half:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_half:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic half %val, ptr %ptr seq_cst, align 2
+ ret void
+}
+
+define void @store_seq_cst_bfloat(ptr %ptr, bfloat %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_bfloat:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur h0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_bfloat:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlrh w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_bfloat:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlrh w1, [x0]
+; NOFP-NEXT: ret
+ store atomic bfloat %val, ptr %ptr seq_cst, align 2
+ ret void
+}
+
+define void @store_seq_cst_float(ptr %ptr, float %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_float:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur s0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_float:
+; NO-RCPC3: fmov w8, s0
+; NO-RCPC3-NEXT: stlr w8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_float:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr w1, [x0]
+; NOFP-NEXT: ret
+ store atomic float %val, ptr %ptr seq_cst, align 4
+ ret void
+}
+
+define void @store_seq_cst_double(ptr %ptr, double %val) nounwind {
+; RCPC3-LABEL: store_seq_cst_double:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+; NO-RCPC3-LABEL: store_seq_cst_double:
+; NO-RCPC3: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
+; NOFP-LABEL: store_seq_cst_double:
+; NOFP: // %bb.0:
+; NOFP-NEXT: stlr x1, [x0]
+; NOFP-NEXT: ret
+ store atomic double %val, ptr %ptr seq_cst, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/rcpc3.ll b/llvm/test/CodeGen/AArch64/rcpc3.ll
index 45770337c13208..5682f8f504881d 100644
--- a/llvm/test/CodeGen/AArch64/rcpc3.ll
+++ b/llvm/test/CodeGen/AArch64/rcpc3.ll
@@ -214,26 +214,34 @@ define hidden void @test_stl1_1xdouble_lane0(ptr nocapture noundef writeonly %a,
ret void
}
-; The remaining tests do not have any particular RCPC3-specific codegen:
-
; load-acquire a plain non-vector double value
define hidden double @test_double_load(ptr nocapture noundef readonly %a) local_unnamed_addr {
-; BOTH-LABEL: test_double_load:
-; BOTH: // %bb.0:
-; BOTH-NEXT: ldapr x8, [x0]
-; BOTH-NEXT: fmov d0, x8
-; BOTH-NEXT: ret
+; RCPC3-LABEL: test_double_load:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: ldapur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_double_load:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: ldapr x8, [x0]
+; NO-RCPC3-NEXT: fmov d0, x8
+; NO-RCPC3-NEXT: ret
%1 = load atomic double, ptr %a acquire, align 8
ret double %1
}
; store-release a plain non-vector double value
define hidden void @test_double_store(ptr nocapture noundef writeonly %a, double noundef %b) local_unnamed_addr {
-; BOTH-LABEL: test_double_store:
-; BOTH: // %bb.0:
-; BOTH-NEXT: fmov x8, d0
-; BOTH-NEXT: stlr x8, [x0]
-; BOTH-NEXT: ret
+; RCPC3-LABEL: test_double_store:
+; RCPC3: // %bb.0:
+; RCPC3-NEXT: stlur d0, [x0]
+; RCPC3-NEXT: ret
+;
+; NO-RCPC3-LABEL: test_double_store:
+; NO-RCPC3: // %bb.0:
+; NO-RCPC3-NEXT: fmov x8, d0
+; NO-RCPC3-NEXT: stlr x8, [x0]
+; NO-RCPC3-NEXT: ret
store atomic double %b, ptr %a release, align 8
ret void
}
More information about the llvm-branch-commits
mailing list