[llvm] [AArch64] Add support for 16/32/64-bit floating-point atomic read-modify-write ops (PR #125686)

Jonathan Thackray via llvm-commits llvm-commits at lists.llvm.org
Fri Feb 7 09:29:38 PST 2025


https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/125686

>From 74f93aa6ead8a101f81fe508664be6e86efdf9a9 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 30 Jan 2025 11:57:22 +0000
Subject: [PATCH 1/2] [AArch64] Add support for 16/32/64-bit floating-point
 atomic read-modify-write ops

Add support for 16/32/64-bit floating-point atomic read-modify-write
operations (FADD, FMAX, FMIN) using LDFADD, LDFMAX, LDFMIN atomic
instructions.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  22 +
 .../lib/Target/AArch64/AArch64InstrAtomics.td |  10 +
 .../lib/Target/AArch64/AArch64InstrFormats.td |  26 +
 .../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 949 +++++++++++++++++
 .../Atomics/aarch64_be-atomicrmw-lsfe.ll      | 964 ++++++++++++++++++
 .../CodeGen/AArch64/Atomics/generate-tests.py |  39 +-
 6 files changed, 2006 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
 create mode 100644 llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 84f6d421b70f961..de7180d78c1326e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -974,6 +974,20 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 #undef LCALLNAME5
   }
 
+  if (Subtarget->outlineAtomics() && !Subtarget->hasLSFE()) {
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f64, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f64, LibCall);
+
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f16, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f32, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f64, LibCall);
+  }
+
   if (Subtarget->hasLSE128()) {
     // Custom lowering because i128 is not legal. Must be replaced by 2x64
     // values. ATOMIC_LOAD_AND also needs op legalisation to emit LDCLRP.
@@ -27825,6 +27839,14 @@ AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   if (CanUseLSE128)
     return AtomicExpansionKind::None;
 
+  // Add support for LDFADD and friends
+  bool CanUseAtomicFP =
+      Subtarget->hasLSFE() && (AI->getOperation() == AtomicRMWInst::FAdd ||
+                               AI->getOperation() == AtomicRMWInst::FMax ||
+                               AI->getOperation() == AtomicRMWInst::FMin);
+  if (CanUseAtomicFP)
+    return AtomicExpansionKind::None;
+
   // Nand is not supported in LSE.
   // Leave 128 bits to LLSC or CmpXChg.
   if (AI->getOperation() != AtomicRMWInst::Nand && Size < 128 &&
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 5e6db9d007a5557..045bebf15a701ab 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -543,6 +543,16 @@ let Predicates = [HasLSE] in {
   defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
 }
 
+defm atomic_load_fadd  : binary_atomic_op_fp<atomic_load_fadd>;
+defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
+defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
+
+let Predicates = [HasLSFE] in {
+  defm : LDFPOPregister_patterns<"LDFADD", "atomic_load_fadd">;
+  defm : LDFPOPregister_patterns<"LDFMAX", "atomic_load_fmax">;
+  defm : LDFPOPregister_patterns<"LDFMIN", "atomic_load_fmin">;
+}
+
 // v8.9a/v9.4a FEAT_LRCPC patterns
 let Predicates = [HasRCPC3, HasNEON] in {
   // LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 3bb5d3cb4d09def..810255eabfded8b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12474,6 +12474,32 @@ multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
                         (i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
 }
 
+let Predicates = [HasLSFE] in
+multiclass LDFPOPregister_patterns_ord_dag<string inst, string suffix, string op,
+                                         ValueType vt, dag SrcRHS, dag DstRHS> {
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") FPR64:$Rn, SrcRHS),
+            (!cast<Instruction>(inst # suffix) DstRHS, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") FPR64:$Rn, SrcRHS),
+            (!cast<Instruction>(inst # "A" # suffix) DstRHS, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") FPR64:$Rn, SrcRHS),
+            (!cast<Instruction>(inst # "L" # suffix) DstRHS, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") FPR64:$Rn, SrcRHS),
+            (!cast<Instruction>(inst # "AL" # suffix) DstRHS, FPR64:$Rn)>;
+  def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") FPR64:$Rn, SrcRHS),
+            (!cast<Instruction>(inst # "AL" # suffix) DstRHS, FPR64:$Rn)>;
+}
+
+multiclass LDFPOPregister_patterns_ord<string inst, string suffix, string op,
+                                     ValueType vt, dag RHS> {
+  defm : LDFPOPregister_patterns_ord_dag<inst, suffix, op, vt, RHS, RHS>;
+}
+
+multiclass LDFPOPregister_patterns<string inst, string op> {
+  defm : LDFPOPregister_patterns_ord<inst, "H", op, f16, (f16 FPR16:$Rm)>;
+  defm : LDFPOPregister_patterns_ord<inst, "S", op, f32, (f32 FPR32:$Rm)>;
+  defm : LDFPOPregister_patterns_ord<inst, "D", op, f64, (f64 FPR64:$Rm)>;
+}
+
 let Predicates = [HasLSE] in
 multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
                                         ValueType vt, dag OLD, dag NEW> {
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
new file mode 100644
index 000000000000000..3d6ad81ea674cc5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -0,0 +1,949 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; CHECK:    ldfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; CHECK:    ldfadda h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release:
+; CHECK:    ldfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; CHECK:    ldfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; CHECK:    ldfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; CHECK:    ldfadd s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; CHECK:    ldfadda s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release:
+; CHECK:    ldfaddl s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; CHECK:    ldfaddal s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; CHECK:    ldfaddal s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; CHECK:    ldfadd d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; CHECK:    ldfadda d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release:
+; CHECK:    ldfaddl d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; CHECK:    ldfaddal d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; CHECK:    ldfaddal d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; CHECK:    ldfmax h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; CHECK:    ldfmaxa h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
+; CHECK:    ldfmaxl h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; CHECK:    ldfmaxal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; CHECK:    ldfmaxal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; CHECK:    ldfmax s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; CHECK:    ldfmaxa s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
+; CHECK:    ldfmaxl s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; CHECK:    ldfmaxal s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; CHECK:    ldfmaxal s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; CHECK:    ldfmax d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; CHECK:    ldfmaxa d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
+; CHECK:    ldfmaxl d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; CHECK:    ldfmaxal d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; CHECK:    ldfmaxal d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; CHECK:    ldfmin h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; CHECK:    ldfmina h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
+; CHECK:    ldfminl h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; CHECK:    ldfminal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; CHECK:    ldfminal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; CHECK:    ldfmin s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; CHECK:    ldfmina s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
+; CHECK:    ldfminl s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; CHECK:    ldfminal s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; CHECK:    ldfminal s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; CHECK:    ldfmin d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; CHECK:    ldfmina d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
+; CHECK:    ldfminl d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; CHECK:    ldfminal d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; CHECK:    ldfminal d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
new file mode 100644
index 000000000000000..d6ae20f4f18a1a4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -0,0 +1,964 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; CHECK:    ldfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; CHECK:    ldfadda h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release:
+; CHECK:    ldfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; CHECK:    ldfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; CHECK:    ldfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; CHECK:    ldfadd s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; CHECK:    ldfadda s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release:
+; CHECK:    ldfaddl s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; CHECK:    ldfaddal s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; CHECK:    ldfaddal s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; CHECK:    ldfadd d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; CHECK:    ldfadda d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release:
+; CHECK:    ldfaddl d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; CHECK:    ldfaddal d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; CHECK:    ldfaddal d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; CHECK:    ldfmax h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; CHECK:    ldfmaxa h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
+; CHECK:    ldfmaxl h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; CHECK:    ldfmaxal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; CHECK:    ldfmaxal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; CHECK:    ldfmax s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; CHECK:    ldfmaxa s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
+; CHECK:    ldfmaxl s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; CHECK:    ldfmaxal s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; CHECK:    ldfmaxal s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; CHECK:    ldfmax d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; CHECK:    ldfmaxa d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
+; CHECK:    ldfmaxl d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; CHECK:    ldfmaxal d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; CHECK:    ldfmaxal d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; CHECK:    ldfmin h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; CHECK:    ldfmina h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
+; CHECK:    ldfminl h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; CHECK:    ldfminal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; CHECK:    ldfminal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; CHECK:    ldfmin s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; CHECK:    ldfmina s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
+; CHECK:    ldfminl s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; CHECK:    ldfminal s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; CHECK:    ldfminal s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; CHECK:    ldfmin d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; CHECK:    ldfmina d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
+; CHECK:    ldfminl d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; CHECK:    ldfminal d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; CHECK:    ldfminal d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index ecda5fd69ca5d93..2f74254e32dcae9 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -29,6 +29,20 @@ def __str__(self) -> str:
         return self.name
 
 
+# FP Type name size
+class FPType(enum.Enum):
+    # Value is the size in bytes
+    half = 2
+    float = 4
+    double = 8
+
+    def align(self, aligned: bool) -> int:
+        return self.value if aligned else 1
+
+    def __str__(self) -> str:
+        return self.name
+
+
 # Is this an aligned or unaligned access?
 class Aligned(enum.Enum):
     aligned = True
@@ -111,6 +125,7 @@ class Feature(enum.Flag):
     v8_1a = enum.auto()  # -mattr=+v8.1a, mandatory FEAT_LOR, FEAT_LSE
     rcpc = enum.auto()  # FEAT_LRCPC
     lse2 = enum.auto()  # FEAT_LSE2
+    lsfe = enum.auto()  # FEAT_LSFE
     outline_atomics = enum.auto()  # -moutline-atomics
     rcpc3 = enum.auto()  # FEAT_LSE2 + FEAT_LRCPC3
     lse2_lse128 = enum.auto()  # FEAT_LSE2 + FEAT_LSE128
@@ -125,6 +140,8 @@ def mattr(self):
             return "+lse2,+rcpc3"
         if self == Feature.lse2_lse128:
             return "+lse2,+lse128"
+        if self == Feature.lsfe:
+            return "+lsfe"
         return "+" + self.name
 
 
@@ -142,11 +159,18 @@ def mattr(self):
     "umin",
 ]
 
+FP_ATOMICRMW_OPS = [
+    "fadd",
+    "fsub",
+    "fmax",
+    "fmin",
+]
+
 
-def all_atomicrmw(f):
-    for op in ATOMICRMW_OPS:
+def all_atomicrmw(f, datatype, atomicrmw_ops):
+    for op in atomicrmw_ops:
         for aligned in Aligned:
-            for ty in Type:
+            for ty in datatype:
                 for ordering in ATOMICRMW_ORDERS:
                     name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}"
                     instr = "atomicrmw"
@@ -286,7 +310,14 @@ def write_lit_tests():
             with open(f"{triple}-atomicrmw-{feat.name}.ll", "w") as f:
                 filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
                 header(f, triple, [feat], filter_args)
-                all_atomicrmw(f)
+                if feat != Feature.lsfe:
+                    all_atomicrmw(f, Type, ATOMICRMW_OPS)
+                else:
+                    all_atomicrmw(f, FPType, FP_ATOMICRMW_OPS)
+
+            # Floating point atomics only supported for atomicrmw currently
+            if feat == Feature.lsfe:
+                continue
 
             with open(f"{triple}-cmpxchg-{feat.name}.ll", "w") as f:
                 filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'

>From 02d9090011481152477eaed9688ee23b5874aadd Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 7 Feb 2025 17:22:39 +0000
Subject: [PATCH 2/2] fixup! Also add support for lowering to
 LDBFADD/LDBFMAX/LDBFMIN

---
 .../Target/AArch64/AArch64ISelLowering.cpp    |   3 +
 .../lib/Target/AArch64/AArch64InstrAtomics.td |  10 +-
 .../lib/Target/AArch64/AArch64InstrFormats.td |   4 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   2 +-
 .../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 540 +++++++++++++++++-
 .../Atomics/aarch64_be-atomicrmw-lsfe.ll      | 540 +++++++++++++++++-
 .../CodeGen/AArch64/Atomics/generate-tests.py |  67 ++-
 7 files changed, 1071 insertions(+), 95 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index de7180d78c1326e..68953428921ac92 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -978,14 +978,17 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FADD, MVT::bf16, LibCall);
 
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMAX, MVT::bf16, LibCall);
 
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f16, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f32, LibCall);
     setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::f64, LibCall);
+    setOperationAction(ISD::ATOMIC_LOAD_FMIN, MVT::bf16, LibCall);
   }
 
   if (Subtarget->hasLSE128()) {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 045bebf15a701ab..2d7a9d6f00bd0e9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -548,9 +548,13 @@ defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
 defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
 
 let Predicates = [HasLSFE] in {
-  defm : LDFPOPregister_patterns<"LDFADD", "atomic_load_fadd">;
-  defm : LDFPOPregister_patterns<"LDFMAX", "atomic_load_fmax">;
-  defm : LDFPOPregister_patterns<"LDFMIN", "atomic_load_fmin">;
+  defm : LDFPOPregister_patterns<"LDFADD",   "atomic_load_fadd">;
+  defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;
+  defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;
+
+  defm : LDBFPOPregister_patterns<"LDBFADD",   "atomic_load_fadd">;
+  defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;
+  defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;
 }
 
 // v8.9a/v9.4a FEAT_LRCPC patterns
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 810255eabfded8b..02f8ac8784b6169 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12500,6 +12500,10 @@ multiclass LDFPOPregister_patterns<string inst, string op> {
   defm : LDFPOPregister_patterns_ord<inst, "D", op, f64, (f64 FPR64:$Rm)>;
 }
 
+multiclass LDBFPOPregister_patterns<string inst, string op> {
+  defm : LDFPOPregister_patterns_ord<inst, "", op, bf16, (bf16 FPR16:$Rm)>;
+}
+
 let Predicates = [HasLSE] in
 multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
                                         ValueType vt, dag OLD, dag NEW> {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 3c57ba414b2bf07..5576a5136ff8a32 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -10565,7 +10565,7 @@ let Predicates = [HasLSFE] in {
   defm LDFMAXNML  : AtomicFPLoad<0b01, 0b110, "ldfmaxnml">;
   defm LDFMINNMA  : AtomicFPLoad<0b10, 0b111, "ldfminnma">;
   defm LDFMINNMAL : AtomicFPLoad<0b11, 0b111, "ldfminnmal">;
-  defm LDFMINMN   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
+  defm LDFMINNM   : AtomicFPLoad<0b00, 0b111, "ldfminnm">;
   defm LDFMINNML  : AtomicFPLoad<0b01, 0b111, "ldfminnml">;
   // BFloat16
   def LDBFADDA    : BaseAtomicFPLoad<FPR16, 0b00, 0b10, 0b000, "ldbfadda">;
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 3d6ad81ea674cc5..666501c937e6520 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -73,6 +73,41 @@ define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; CHECK:    ldbfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; CHECK:    ldbfadda h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; CHECK:    ldbfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; CHECK:    ldbfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; CHECK:    ldbfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
 ; CHECK:    ldfadd d0, d0, [x0]
@@ -178,6 +213,76 @@ define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -353,6 +458,101 @@ define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
 ; -O0:    ldaxr x0, [x9]
@@ -493,6 +693,76 @@ define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -530,105 +800,140 @@ define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; CHECK:    ldfmax h0, h0, [x0]
+; CHECK:    ldfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; CHECK:    ldfmaxa h0, h0, [x0]
+; CHECK:    ldfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
-; CHECK:    ldfmaxl h0, h0, [x0]
+; CHECK:    ldfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; CHECK:    ldfmaxal h0, h0, [x0]
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; CHECK:    ldfmaxal h0, h0, [x0]
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; CHECK:    ldfmax s0, s0, [x0]
+; CHECK:    ldfmaxnm s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; CHECK:    ldfmaxa s0, s0, [x0]
+; CHECK:    ldfmaxnma s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
-; CHECK:    ldfmaxl s0, s0, [x0]
+; CHECK:    ldfmaxnml s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; CHECK:    ldfmaxal s0, s0, [x0]
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; CHECK:    ldfmaxal s0, s0, [x0]
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxnma h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; CHECK:    ldfmax d0, d0, [x0]
+; CHECK:    ldfmaxnm d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; CHECK:    ldfmaxa d0, d0, [x0]
+; CHECK:    ldfmaxnma d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
-; CHECK:    ldfmaxl d0, d0, [x0]
+; CHECK:    ldfmaxnml d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; CHECK:    ldfmaxal d0, d0, [x0]
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; CHECK:    ldfmaxal d0, d0, [x0]
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -703,6 +1008,76 @@ define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -740,105 +1115,140 @@ define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; CHECK:    ldfmin h0, h0, [x0]
+; CHECK:    ldfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; CHECK:    ldfmina h0, h0, [x0]
+; CHECK:    ldfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
-; CHECK:    ldfminl h0, h0, [x0]
+; CHECK:    ldfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; CHECK:    ldfminal h0, h0, [x0]
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; CHECK:    ldfminal h0, h0, [x0]
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; CHECK:    ldfmin s0, s0, [x0]
+; CHECK:    ldfminnm s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; CHECK:    ldfmina s0, s0, [x0]
+; CHECK:    ldfminnma s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
-; CHECK:    ldfminl s0, s0, [x0]
+; CHECK:    ldfminnml s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; CHECK:    ldfminal s0, s0, [x0]
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; CHECK:    ldfminal s0, s0, [x0]
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; CHECK:    ldbfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; CHECK:    ldbfminnma h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; CHECK:    ldbfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminnmal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminnmal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; CHECK:    ldfmin d0, d0, [x0]
+; CHECK:    ldfminnm d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; CHECK:    ldfmina d0, d0, [x0]
+; CHECK:    ldfminnma d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
-; CHECK:    ldfminl d0, d0, [x0]
+; CHECK:    ldfminnml d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; CHECK:    ldfminal d0, d0, [x0]
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; CHECK:    ldfminal d0, d0, [x0]
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -913,6 +1323,76 @@ define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index d6ae20f4f18a1a4..8a0a92dee0b2ef0 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -73,6 +73,41 @@ define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; CHECK:    ldbfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; CHECK:    ldbfadda h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; CHECK:    ldbfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; CHECK:    ldbfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; CHECK:    ldbfaddal h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic:
 ; CHECK:    ldfadd d0, d0, [x0]
@@ -178,6 +213,76 @@ define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -363,6 +468,101 @@ define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
 ; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
 ; -O0:    ldaxr x9, [x11]
@@ -508,6 +708,76 @@ define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -545,105 +815,140 @@ define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic:
-; CHECK:    ldfmax h0, h0, [x0]
+; CHECK:    ldfmaxnm h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
-; CHECK:    ldfmaxa h0, h0, [x0]
+; CHECK:    ldfmaxnma h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_release:
-; CHECK:    ldfmaxl h0, h0, [x0]
+; CHECK:    ldfmaxnml h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
-; CHECK:    ldfmaxal h0, h0, [x0]
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
-; CHECK:    ldfmaxal h0, h0, [x0]
+; CHECK:    ldfmaxnmal h0, h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic:
-; CHECK:    ldfmax s0, s0, [x0]
+; CHECK:    ldfmaxnm s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
-; CHECK:    ldfmaxa s0, s0, [x0]
+; CHECK:    ldfmaxnma s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_release:
-; CHECK:    ldfmaxl s0, s0, [x0]
+; CHECK:    ldfmaxnml s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
-; CHECK:    ldfmaxal s0, s0, [x0]
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
-; CHECK:    ldfmaxal s0, s0, [x0]
+; CHECK:    ldfmaxnmal s0, s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; CHECK:    ldbfmaxnma h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; CHECK:    ldbfmaxnmal h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic:
-; CHECK:    ldfmax d0, d0, [x0]
+; CHECK:    ldfmaxnm d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
-; CHECK:    ldfmaxa d0, d0, [x0]
+; CHECK:    ldfmaxnma d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_release:
-; CHECK:    ldfmaxl d0, d0, [x0]
+; CHECK:    ldfmaxnml d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
-; CHECK:    ldfmaxal d0, d0, [x0]
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
-; CHECK:    ldfmaxal d0, d0, [x0]
+; CHECK:    ldfmaxnmal d0, d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -718,6 +1023,76 @@ define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
@@ -755,105 +1130,140 @@ define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, doubl
 
 define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic:
-; CHECK:    ldfmin h0, h0, [x0]
+; CHECK:    ldfminnm h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
-; CHECK:    ldfmina h0, h0, [x0]
+; CHECK:    ldfminnma h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_release:
-; CHECK:    ldfminl h0, h0, [x0]
+; CHECK:    ldfminnml h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
-; CHECK:    ldfminal h0, h0, [x0]
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
     ret half %r
 }
 
 define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
-; CHECK:    ldfminal h0, h0, [x0]
+; CHECK:    ldfminnmal h0, h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
     ret half %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic:
-; CHECK:    ldfmin s0, s0, [x0]
+; CHECK:    ldfminnm s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
-; CHECK:    ldfmina s0, s0, [x0]
+; CHECK:    ldfminnma s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_release:
-; CHECK:    ldfminl s0, s0, [x0]
+; CHECK:    ldfminnml s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
-; CHECK:    ldfminal s0, s0, [x0]
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
     ret float %r
 }
 
 define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
-; CHECK:    ldfminal s0, s0, [x0]
+; CHECK:    ldfminnmal s0, s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; CHECK:    ldbfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; CHECK:    ldbfminnma h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; CHECK:    ldbfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; CHECK:    ldbfminnmal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 4
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; CHECK:    ldbfminnmal h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 4
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic:
-; CHECK:    ldfmin d0, d0, [x0]
+; CHECK:    ldfminnm d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
-; CHECK:    ldfmina d0, d0, [x0]
+; CHECK:    ldfminnma d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_release:
-; CHECK:    ldfminl d0, d0, [x0]
+; CHECK:    ldfminnml d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
-; CHECK:    ldfminal d0, d0, [x0]
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
     ret double %r
 }
 
 define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
-; CHECK:    ldfminal d0, d0, [x0]
+; CHECK:    ldfminnmal d0, d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
     ret double %r
 }
@@ -928,6 +1338,76 @@ define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
 define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
 ; CHECK:    bl __atomic_compare_exchange
diff --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index 2f74254e32dcae9..2bbcd2032a8995e 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -13,34 +13,35 @@
 ]
 
 
-# Type name size
-class Type(enum.Enum):
-    # Value is the size in bytes
-    i8 = 1
-    i16 = 2
-    i32 = 4
-    i64 = 8
-    i128 = 16
-
-    def align(self, aligned: bool) -> int:
-        return self.value if aligned else 1
+class byteSizes:
+    def __init__(self):
+        self._data = {}
 
-    def __str__(self) -> str:
-        return self.name
+    def __setitem__(self, k, v):
+        self._data[k] = v
 
+    def __getitem__(self, k):
+        return self._data[k]
 
-# FP Type name size
-class FPType(enum.Enum):
-    # Value is the size in bytes
-    half = 2
-    float = 4
-    double = 8
+    def __iter__(self):
+        return iter(self._data.items())
 
-    def align(self, aligned: bool) -> int:
-        return self.value if aligned else 1
+def align(val, aligned: bool) -> int:
+   return val if aligned else 1
 
-    def __str__(self) -> str:
-        return self.name
+# Value is the size in bytes
+Type = byteSizes()
+Type['i8'] = 1
+Type['i16'] = 2
+Type['i32'] = 4
+Type['i64'] = 8
+Type['i128'] = 16
+
+FPType = byteSizes()
+FPType['half'] = 2
+FPType['float'] = 4
+FPType['bfloat'] = 4
+FPType['double'] = 8
 
 
 # Is this an aligned or unaligned access?
@@ -170,7 +171,8 @@ def mattr(self):
 def all_atomicrmw(f, datatype, atomicrmw_ops):
     for op in atomicrmw_ops:
         for aligned in Aligned:
-            for ty in datatype:
+            for ty, val in datatype:
+                alignval = align(val, aligned)
                 for ordering in ATOMICRMW_ORDERS:
                     name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}"
                     instr = "atomicrmw"
@@ -178,7 +180,7 @@ def all_atomicrmw(f, datatype, atomicrmw_ops):
                         textwrap.dedent(
                             f"""
                         define dso_local {ty} @{name}(ptr %ptr, {ty} %value) {{
-                            %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {ty.align(aligned)}
+                            %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval}
                             ret {ty} %r
                         }}
                     """
@@ -188,7 +190,8 @@ def all_atomicrmw(f, datatype, atomicrmw_ops):
 
 def all_load(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for ordering in ATOMIC_LOAD_ORDERS:
                 for const in [False, True]:
                     name = f"load_atomic_{ty}_{aligned}_{ordering}"
@@ -200,7 +203,7 @@ def all_load(f):
                         textwrap.dedent(
                             f"""
                         define dso_local {ty} @{name}({arg}) {{
-                            %r = {instr} {ty}, ptr %ptr {ordering}, align {ty.align(aligned)}
+                            %r = {instr} {ty}, ptr %ptr {ordering}, align {alignval}
                             ret {ty} %r
                         }}
                     """
@@ -210,7 +213,8 @@ def all_load(f):
 
 def all_store(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for ordering in ATOMIC_STORE_ORDERS:  # FIXME stores
                 name = f"store_atomic_{ty}_{aligned}_{ordering}"
                 instr = "store atomic"
@@ -218,7 +222,7 @@ def all_store(f):
                     textwrap.dedent(
                         f"""
                     define dso_local void @{name}({ty} %value, ptr %ptr) {{
-                        {instr} {ty} %value, ptr %ptr {ordering}, align {ty.align(aligned)}
+                        {instr} {ty} %value, ptr %ptr {ordering}, align {alignval}
                         ret void
                     }}
                 """
@@ -228,7 +232,8 @@ def all_store(f):
 
 def all_cmpxchg(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for success_ordering in CMPXCHG_SUCCESS_ORDERS:
                 for failure_ordering in CMPXCHG_FAILURE_ORDERS:
                     for weak in [False, True]:
@@ -241,7 +246,7 @@ def all_cmpxchg(f):
                             textwrap.dedent(
                                 f"""
                             define dso_local {ty} @{name}({ty} %expected, {ty} %new, ptr %ptr) {{
-                                %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {ty.align(aligned)}
+                                %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {alignval}
                                 %r = extractvalue {{ {ty}, i1 }} %pair, 0
                                 ret {ty} %r
                             }}



More information about the llvm-commits mailing list