[llvm] [AArch64][CodeGen] Add patterns for +lsfe atomic stores (PR #131174)

Kerry McLaughlin via llvm-commits llvm-commits at lists.llvm.org
Thu Mar 13 10:50:20 PDT 2025


https://github.com/kmclaughlin-arm updated https://github.com/llvm/llvm-project/pull/131174

>From ba0dc8f83b2bddbb5858ba46943512bfdb9aea33 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 13 Feb 2025 10:17:03 +0000
Subject: [PATCH 1/3] Update generate-tests.py script to add +lsfe atomicrmw
 tests where the result is unused.

---
 .../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 168 ++++++++++++++++++
 .../Atomics/aarch64_be-atomicrmw-lsfe.ll      | 168 ++++++++++++++++++
 .../CodeGen/AArch64/Atomics/generate-tests.py |  26 ++-
 3 files changed, 359 insertions(+), 3 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index fc9a126f79a83..5b841d9331ce1 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -10,6 +10,13 @@ define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fadd_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic_to_store:
+; CHECK:    ldfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
 ; CHECK:    ldfadda h0, h0, [x0]
@@ -24,6 +31,13 @@ define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fadd_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release_to_store:
+; CHECK:    ldfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
 ; CHECK:    ldfaddal h0, h0, [x0]
@@ -45,6 +59,13 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fadd_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; CHECK:    ldbfadda h0, h0, [x0]
@@ -59,6 +80,13 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fadd_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release_to_store:
+; CHECK:    ldbfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfaddal h0, h0, [x0]
@@ -80,6 +108,13 @@ define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fadd_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic_to_store:
+; CHECK:    ldfadd s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
 ; CHECK:    ldfadda s0, s0, [x0]
@@ -94,6 +129,13 @@ define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fadd_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release_to_store:
+; CHECK:    ldfaddl s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
 ; CHECK:    ldfaddal s0, s0, [x0]
@@ -115,6 +157,13 @@ define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fadd_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic_to_store:
+; CHECK:    ldfadd d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
 ; CHECK:    ldfadda d0, d0, [x0]
@@ -129,6 +178,13 @@ define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fadd_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release_to_store:
+; CHECK:    ldfaddl d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
 ; CHECK:    ldfaddal d0, d0, [x0]
@@ -805,6 +861,13 @@ define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmax_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
 ; CHECK:    ldfmaxnma h0, h0, [x0]
@@ -819,6 +882,13 @@ define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmax_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release_to_store:
+; CHECK:    ldfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal h0, h0, [x0]
@@ -840,6 +910,13 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmax_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; CHECK:    ldbfmaxnma h0, h0, [x0]
@@ -854,6 +931,13 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmax_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release_to_store:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfmaxnmal h0, h0, [x0]
@@ -875,6 +959,13 @@ define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmax_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
 ; CHECK:    ldfmaxnma s0, s0, [x0]
@@ -889,6 +980,13 @@ define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmax_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release_to_store:
+; CHECK:    ldfmaxnml s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal s0, s0, [x0]
@@ -910,6 +1008,13 @@ define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmax_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
 ; CHECK:    ldfmaxnma d0, d0, [x0]
@@ -924,6 +1029,13 @@ define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmax_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release_to_store:
+; CHECK:    ldfmaxnml d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal d0, d0, [x0]
@@ -1120,6 +1232,13 @@ define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmin_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic_to_store:
+; CHECK:    ldfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
 ; CHECK:    ldfminnma h0, h0, [x0]
@@ -1134,6 +1253,13 @@ define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmin_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release_to_store:
+; CHECK:    ldfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
 ; CHECK:    ldfminnmal h0, h0, [x0]
@@ -1155,6 +1281,13 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmin_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; CHECK:    ldbfminnma h0, h0, [x0]
@@ -1169,6 +1302,13 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmin_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release_to_store:
+; CHECK:    ldbfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfminnmal h0, h0, [x0]
@@ -1190,6 +1330,13 @@ define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmin_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic_to_store:
+; CHECK:    ldfminnm s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
 ; CHECK:    ldfminnma s0, s0, [x0]
@@ -1204,6 +1351,13 @@ define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmin_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release_to_store:
+; CHECK:    ldfminnml s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
 ; CHECK:    ldfminnmal s0, s0, [x0]
@@ -1225,6 +1379,13 @@ define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmin_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic_to_store:
+; CHECK:    ldfminnm d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
 ; CHECK:    ldfminnma d0, d0, [x0]
@@ -1239,6 +1400,13 @@ define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmin_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release_to_store:
+; CHECK:    ldfminnml d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
 ; CHECK:    ldfminnmal d0, d0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index a22cc5806d86d..47244ccbf7ef0 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -10,6 +10,13 @@ define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fadd_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic_to_store:
+; CHECK:    ldfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_acquire:
 ; CHECK:    ldfadda h0, h0, [x0]
@@ -24,6 +31,13 @@ define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fadd_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_aligned_release_to_store:
+; CHECK:    ldfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
 ; CHECK:    ldfaddal h0, h0, [x0]
@@ -45,6 +59,13 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fadd_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfadd h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
 ; CHECK:    ldbfadda h0, h0, [x0]
@@ -59,6 +80,13 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fadd_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release_to_store:
+; CHECK:    ldbfaddl h0, h0, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfaddal h0, h0, [x0]
@@ -80,6 +108,13 @@ define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fadd_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic_to_store:
+; CHECK:    ldfadd s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_acquire:
 ; CHECK:    ldfadda s0, s0, [x0]
@@ -94,6 +129,13 @@ define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fadd_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_aligned_release_to_store:
+; CHECK:    ldfaddl s0, s0, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
 ; CHECK:    ldfaddal s0, s0, [x0]
@@ -115,6 +157,13 @@ define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fadd_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic_to_store:
+; CHECK:    ldfadd d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_acquire:
 ; CHECK:    ldfadda d0, d0, [x0]
@@ -129,6 +178,13 @@ define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fadd_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_aligned_release_to_store:
+; CHECK:    ldfaddl d0, d0, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
 ; CHECK:    ldfaddal d0, d0, [x0]
@@ -820,6 +876,13 @@ define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmax_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acquire:
 ; CHECK:    ldfmaxnma h0, h0, [x0]
@@ -834,6 +897,13 @@ define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmax_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_aligned_release_to_store:
+; CHECK:    ldfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal h0, h0, [x0]
@@ -855,6 +925,13 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmax_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfmaxnm h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
 ; CHECK:    ldbfmaxnma h0, h0, [x0]
@@ -869,6 +946,13 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmax_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release_to_store:
+; CHECK:    ldbfmaxnml h0, h0, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfmaxnmal h0, h0, [x0]
@@ -890,6 +974,13 @@ define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmax_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acquire:
 ; CHECK:    ldfmaxnma s0, s0, [x0]
@@ -904,6 +995,13 @@ define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmax_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_aligned_release_to_store:
+; CHECK:    ldfmaxnml s0, s0, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal s0, s0, [x0]
@@ -925,6 +1023,13 @@ define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmax_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic_to_store:
+; CHECK:    ldfmaxnm d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acquire:
 ; CHECK:    ldfmaxnma d0, d0, [x0]
@@ -939,6 +1044,13 @@ define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmax_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_aligned_release_to_store:
+; CHECK:    ldfmaxnml d0, d0, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
 ; CHECK:    ldfmaxnmal d0, d0, [x0]
@@ -1135,6 +1247,13 @@ define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %val
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmin_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic_to_store:
+; CHECK:    ldfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acquire:
 ; CHECK:    ldfminnma h0, h0, [x0]
@@ -1149,6 +1268,13 @@ define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value
     ret half %r
 }
 
+define dso_local void @atomicrmw_fmin_half_aligned_release_to_store(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_aligned_release_to_store:
+; CHECK:    ldfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret void
+}
+
 define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
 ; CHECK:    ldfminnmal h0, h0, [x0]
@@ -1170,6 +1296,13 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmin_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic_to_store:
+; CHECK:    ldbfminnm h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
 ; CHECK:    ldbfminnma h0, h0, [x0]
@@ -1184,6 +1317,13 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
     ret bfloat %r
 }
 
+define dso_local void @atomicrmw_fmin_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
+; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release_to_store:
+; CHECK:    ldbfminnml h0, h0, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret void
+}
+
 define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
 ; CHECK:    ldbfminnmal h0, h0, [x0]
@@ -1205,6 +1345,13 @@ define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmin_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic_to_store:
+; CHECK:    ldfminnm s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acquire:
 ; CHECK:    ldfminnma s0, s0, [x0]
@@ -1219,6 +1366,13 @@ define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %va
     ret float %r
 }
 
+define dso_local void @atomicrmw_fmin_float_aligned_release_to_store(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_aligned_release_to_store:
+; CHECK:    ldfminnml s0, s0, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret void
+}
+
 define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
 ; CHECK:    ldfminnmal s0, s0, [x0]
@@ -1240,6 +1394,13 @@ define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, doubl
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmin_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic_to_store:
+; CHECK:    ldfminnm d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acquire:
 ; CHECK:    ldfminnma d0, d0, [x0]
@@ -1254,6 +1415,13 @@ define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double
     ret double %r
 }
 
+define dso_local void @atomicrmw_fmin_double_aligned_release_to_store(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_aligned_release_to_store:
+; CHECK:    ldfminnml d0, d0, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret void
+}
+
 define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
 ; CHECK:    ldfminnmal d0, d0, [x0]
diff --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index f40bbaeb930c0..b995c38463034 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -189,18 +189,26 @@ def relpath():
     return fp
 
 
+def generate_store_opt_test(featname, ordering, op, alignval):
+  if featname != 'lsfe' or op == 'fsub' or alignval == 1:
+    return False
+  if ordering not in [AtomicOrder.monotonic, AtomicOrder.release]:
+    return False
+  return True;
+
+
 def align(val, aligned: bool) -> int:
     return val if aligned else 1
 
 
-def all_atomicrmw(f, datatype, atomicrmw_ops):
+def all_atomicrmw(f, datatype, atomicrmw_ops, featname):
+    instr = "atomicrmw"
     for op in atomicrmw_ops:
         for aligned in Aligned:
             for ty, val in datatype:
                 alignval = align(val, aligned)
                 for ordering in ATOMICRMW_ORDERS:
                     name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}"
-                    instr = "atomicrmw"
                     f.write(
                         textwrap.dedent(
                             f"""
@@ -211,6 +219,18 @@ def all_atomicrmw(f, datatype, atomicrmw_ops):
                     """
                         )
                     )
+                    if generate_store_opt_test(featname, ordering, op, alignval):
+                        name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}_to_store"
+                        f.write(
+                           textwrap.dedent(
+                               f"""
+                           define dso_local void @{name}(ptr %ptr, {ty} %value) {{
+                               %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval}
+                               ret void
+                           }}
+                        """
+                           )
+                        )
 
 
 def all_load(f):
@@ -340,7 +360,7 @@ def write_lit_tests(feature, datatypes, ops):
             with open(f"{triple}-atomicrmw-{feat.name}.ll", "w") as f:
                 filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
                 header(f, triple, [feat], filter_args)
-                all_atomicrmw(f, datatypes, ops)
+                all_atomicrmw(f, datatypes, ops, feat.name)
 
             # Floating point atomics only supported for atomicrmw currently
             if feature.test_scope() == "atomicrmw":

>From aaefe4b5f42e8f446f562577da44b3749ab74ebc Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 13 Feb 2025 10:17:03 +0000
Subject: [PATCH 2/3] [AArch64] Add patterns for +lsfe atomic stores

When FEAT_LSFE is enabled, the ST{B}FADD, ST{B}FMAX and ST{B}FMIN
atomic instructions are available. This adds patterns to match an
atomicrmw fadd, fmin or fmax to these instructions when the result
is unused.
---
 .../lib/Target/AArch64/AArch64InstrAtomics.td | 12 ++---
 .../lib/Target/AArch64/AArch64InstrFormats.td | 32 ++++++++++---
 .../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 48 +++++++++----------
 .../Atomics/aarch64_be-atomicrmw-lsfe.ll      | 48 +++++++++----------
 4 files changed, 80 insertions(+), 60 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2d7a9d6f00bd0..aaf8e1bfc8f0f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -548,13 +548,13 @@ defm atomic_load_fmin  : binary_atomic_op_fp<atomic_load_fmin>;
 defm atomic_load_fmax  : binary_atomic_op_fp<atomic_load_fmax>;
 
 let Predicates = [HasLSFE] in {
-  defm : LDFPOPregister_patterns<"LDFADD",   "atomic_load_fadd">;
-  defm : LDFPOPregister_patterns<"LDFMAXNM", "atomic_load_fmax">;
-  defm : LDFPOPregister_patterns<"LDFMINNM", "atomic_load_fmin">;
+  defm : LDFPOPregister_patterns<"FADD",   "atomic_load_fadd">;
+  defm : LDFPOPregister_patterns<"FMAXNM", "atomic_load_fmax">;
+  defm : LDFPOPregister_patterns<"FMINNM", "atomic_load_fmin">;
 
-  defm : LDBFPOPregister_patterns<"LDBFADD",   "atomic_load_fadd">;
-  defm : LDBFPOPregister_patterns<"LDBFMAXNM", "atomic_load_fmax">;
-  defm : LDBFPOPregister_patterns<"LDBFMINNM", "atomic_load_fmin">;
+  defm : LDBFPOPregister_patterns<"BFADD",   "atomic_load_fadd">;
+  defm : LDBFPOPregister_patterns<"BFMAXNM", "atomic_load_fmax">;
+  defm : LDBFPOPregister_patterns<"BFMINNM", "atomic_load_fmin">;
 }
 
 // v8.9a/v9.4a FEAT_LRCPC patterns
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 255cd0ec5840c..99bd85a448d83 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12493,19 +12493,39 @@ multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
                         (i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
 }
 
+class AtomicStoreMonotonicFrag<SDNode atomic_op> : PatFrag<
+  (ops node:$ptr, node:$value),
+  (atomic_op node:$ptr, node:$value),
+  [{ return SDValue(N, 0).use_empty() &&
+     cast<AtomicSDNode>(N)->getMergedOrdering() == AtomicOrdering::Monotonic; }]>;
+
+class AtomicStoreReleaseFrag<SDNode atomic_op> : PatFrag<
+  (ops node:$ptr, node:$value),
+  (atomic_op node:$ptr, node:$value),
+  [{ return SDValue(N, 0).use_empty() &&
+     cast<AtomicSDNode>(N)->getMergedOrdering() == AtomicOrdering::Release; }]>;
+
 let Predicates = [HasLSFE] in
 multiclass LDFPOPregister_patterns_ord_dag<string inst, string suffix, string op,
-                                         ValueType vt, dag data> {
+                                           ValueType vt, dag data> {
   def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") FPR64:$Rn, data),
-            (!cast<Instruction>(inst # suffix) data, FPR64:$Rn)>;
+            (!cast<Instruction>("LD" # inst # suffix) data, FPR64:$Rn)>;
   def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") FPR64:$Rn, data),
-            (!cast<Instruction>(inst # "A" # suffix) data, FPR64:$Rn)>;
+            (!cast<Instruction>("LD" # inst # "A" # suffix) data, FPR64:$Rn)>;
   def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") FPR64:$Rn, data),
-            (!cast<Instruction>(inst # "L" # suffix) data, FPR64:$Rn)>;
+            (!cast<Instruction>("LD" # inst # "L" # suffix) data, FPR64:$Rn)>;
   def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") FPR64:$Rn, data),
-            (!cast<Instruction>(inst # "AL" # suffix) data, FPR64:$Rn)>;
+            (!cast<Instruction>("LD" # inst # "AL" # suffix) data, FPR64:$Rn)>;
   def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") FPR64:$Rn, data),
-            (!cast<Instruction>(inst # "AL" # suffix) data, FPR64:$Rn)>;
+            (!cast<Instruction>("LD" # inst # "AL" # suffix) data, FPR64:$Rn)>;
+
+  let AddedComplexity = 5 in {
+    def : Pat<(AtomicStoreMonotonicFrag<!cast<SDNode>(op)> FPR64:$Rn, data),
+              (!cast<Instruction>("ST" # inst # suffix) data, FPR64:$Rn)>;
+
+    def : Pat<(AtomicStoreReleaseFrag<!cast<SDNode>(op)> FPR64:$Rn, data),
+              (!cast<Instruction>("ST" # inst # "L" # suffix) data, FPR64:$Rn)>;
+  }
 }
 
 multiclass LDFPOPregister_patterns_ord<string inst, string suffix, string op,
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 5b841d9331ce1..bd9b814ac533e 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -12,7 +12,7 @@ define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fadd_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic_to_store:
-; CHECK:    ldfadd h0, h0, [x0]
+; CHECK:    stfadd h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -33,7 +33,7 @@ define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fadd_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_release_to_store:
-; CHECK:    ldfaddl h0, h0, [x0]
+; CHECK:    stfaddl h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value release, align 2
     ret void
 }
@@ -61,7 +61,7 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fadd_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfadd h0, h0, [x0]
+; CHECK:    stbfadd h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -82,7 +82,7 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fadd_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release_to_store:
-; CHECK:    ldbfaddl h0, h0, [x0]
+; CHECK:    stbfaddl h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -110,7 +110,7 @@ define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fadd_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic_to_store:
-; CHECK:    ldfadd s0, s0, [x0]
+; CHECK:    stfadd s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -131,7 +131,7 @@ define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fadd_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_release_to_store:
-; CHECK:    ldfaddl s0, s0, [x0]
+; CHECK:    stfaddl s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value release, align 4
     ret void
 }
@@ -159,7 +159,7 @@ define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fadd_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic_to_store:
-; CHECK:    ldfadd d0, d0, [x0]
+; CHECK:    stfadd d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -180,7 +180,7 @@ define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fadd_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_release_to_store:
-; CHECK:    ldfaddl d0, d0, [x0]
+; CHECK:    stfaddl d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value release, align 8
     ret void
 }
@@ -863,7 +863,7 @@ define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fmax_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm h0, h0, [x0]
+; CHECK:    stfmaxnm h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -884,7 +884,7 @@ define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fmax_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_release_to_store:
-; CHECK:    ldfmaxnml h0, h0, [x0]
+; CHECK:    stfmaxnml h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret void
 }
@@ -912,7 +912,7 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fmax_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfmaxnm h0, h0, [x0]
+; CHECK:    stbfmaxnm h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -933,7 +933,7 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fmax_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release_to_store:
-; CHECK:    ldbfmaxnml h0, h0, [x0]
+; CHECK:    stbfmaxnml h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -961,7 +961,7 @@ define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fmax_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm s0, s0, [x0]
+; CHECK:    stfmaxnm s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -982,7 +982,7 @@ define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fmax_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_release_to_store:
-; CHECK:    ldfmaxnml s0, s0, [x0]
+; CHECK:    stfmaxnml s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret void
 }
@@ -1010,7 +1010,7 @@ define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fmax_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm d0, d0, [x0]
+; CHECK:    stfmaxnm d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -1031,7 +1031,7 @@ define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fmax_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_release_to_store:
-; CHECK:    ldfmaxnml d0, d0, [x0]
+; CHECK:    stfmaxnml d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret void
 }
@@ -1234,7 +1234,7 @@ define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fmin_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic_to_store:
-; CHECK:    ldfminnm h0, h0, [x0]
+; CHECK:    stfminnm h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -1255,7 +1255,7 @@ define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fmin_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_release_to_store:
-; CHECK:    ldfminnml h0, h0, [x0]
+; CHECK:    stfminnml h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret void
 }
@@ -1283,7 +1283,7 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fmin_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfminnm h0, h0, [x0]
+; CHECK:    stbfminnm h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -1304,7 +1304,7 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fmin_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release_to_store:
-; CHECK:    ldbfminnml h0, h0, [x0]
+; CHECK:    stbfminnml h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -1332,7 +1332,7 @@ define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fmin_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic_to_store:
-; CHECK:    ldfminnm s0, s0, [x0]
+; CHECK:    stfminnm s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -1353,7 +1353,7 @@ define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fmin_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_release_to_store:
-; CHECK:    ldfminnml s0, s0, [x0]
+; CHECK:    stfminnml s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret void
 }
@@ -1381,7 +1381,7 @@ define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fmin_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic_to_store:
-; CHECK:    ldfminnm d0, d0, [x0]
+; CHECK:    stfminnm d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -1402,7 +1402,7 @@ define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fmin_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_release_to_store:
-; CHECK:    ldfminnml d0, d0, [x0]
+; CHECK:    stfminnml d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
index 47244ccbf7ef0..67a5565f31d94 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -12,7 +12,7 @@ define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fadd_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_monotonic_to_store:
-; CHECK:    ldfadd h0, h0, [x0]
+; CHECK:    stfadd h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -33,7 +33,7 @@ define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fadd_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fadd_half_aligned_release_to_store:
-; CHECK:    ldfaddl h0, h0, [x0]
+; CHECK:    stfaddl h0, [x0]
     %r = atomicrmw fadd ptr %ptr, half %value release, align 2
     ret void
 }
@@ -61,7 +61,7 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fadd_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfadd h0, h0, [x0]
+; CHECK:    stbfadd h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -82,7 +82,7 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fadd_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fadd_bfloat_aligned_release_to_store:
-; CHECK:    ldbfaddl h0, h0, [x0]
+; CHECK:    stbfaddl h0, [x0]
     %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -110,7 +110,7 @@ define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fadd_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_monotonic_to_store:
-; CHECK:    ldfadd s0, s0, [x0]
+; CHECK:    stfadd s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -131,7 +131,7 @@ define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fadd_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fadd_float_aligned_release_to_store:
-; CHECK:    ldfaddl s0, s0, [x0]
+; CHECK:    stfaddl s0, [x0]
     %r = atomicrmw fadd ptr %ptr, float %value release, align 4
     ret void
 }
@@ -159,7 +159,7 @@ define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fadd_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_monotonic_to_store:
-; CHECK:    ldfadd d0, d0, [x0]
+; CHECK:    stfadd d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -180,7 +180,7 @@ define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fadd_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fadd_double_aligned_release_to_store:
-; CHECK:    ldfaddl d0, d0, [x0]
+; CHECK:    stfaddl d0, [x0]
     %r = atomicrmw fadd ptr %ptr, double %value release, align 8
     ret void
 }
@@ -878,7 +878,7 @@ define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fmax_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm h0, h0, [x0]
+; CHECK:    stfmaxnm h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -899,7 +899,7 @@ define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fmax_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmax_half_aligned_release_to_store:
-; CHECK:    ldfmaxnml h0, h0, [x0]
+; CHECK:    stfmaxnml h0, [x0]
     %r = atomicrmw fmax ptr %ptr, half %value release, align 2
     ret void
 }
@@ -927,7 +927,7 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fmax_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfmaxnm h0, h0, [x0]
+; CHECK:    stbfmaxnm h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -948,7 +948,7 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fmax_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmax_bfloat_aligned_release_to_store:
-; CHECK:    ldbfmaxnml h0, h0, [x0]
+; CHECK:    stbfmaxnml h0, [x0]
     %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -976,7 +976,7 @@ define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fmax_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm s0, s0, [x0]
+; CHECK:    stfmaxnm s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -997,7 +997,7 @@ define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fmax_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmax_float_aligned_release_to_store:
-; CHECK:    ldfmaxnml s0, s0, [x0]
+; CHECK:    stfmaxnml s0, [x0]
     %r = atomicrmw fmax ptr %ptr, float %value release, align 4
     ret void
 }
@@ -1025,7 +1025,7 @@ define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fmax_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_monotonic_to_store:
-; CHECK:    ldfmaxnm d0, d0, [x0]
+; CHECK:    stfmaxnm d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -1046,7 +1046,7 @@ define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fmax_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmax_double_aligned_release_to_store:
-; CHECK:    ldfmaxnml d0, d0, [x0]
+; CHECK:    stfmaxnml d0, [x0]
     %r = atomicrmw fmax ptr %ptr, double %value release, align 8
     ret void
 }
@@ -1249,7 +1249,7 @@ define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %val
 
 define dso_local void @atomicrmw_fmin_half_aligned_monotonic_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_monotonic_to_store:
-; CHECK:    ldfminnm h0, h0, [x0]
+; CHECK:    stfminnm h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
     ret void
 }
@@ -1270,7 +1270,7 @@ define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value
 
 define dso_local void @atomicrmw_fmin_half_aligned_release_to_store(ptr %ptr, half %value) {
 ; CHECK-LABEL: atomicrmw_fmin_half_aligned_release_to_store:
-; CHECK:    ldfminnml h0, h0, [x0]
+; CHECK:    stfminnml h0, [x0]
     %r = atomicrmw fmin ptr %ptr, half %value release, align 2
     ret void
 }
@@ -1298,7 +1298,7 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
 
 define dso_local void @atomicrmw_fmin_bfloat_aligned_monotonic_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic_to_store:
-; CHECK:    ldbfminnm h0, h0, [x0]
+; CHECK:    stbfminnm h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
     ret void
 }
@@ -1319,7 +1319,7 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
 
 define dso_local void @atomicrmw_fmin_bfloat_aligned_release_to_store(ptr %ptr, bfloat %value) {
 ; CHECK-LABEL: atomicrmw_fmin_bfloat_aligned_release_to_store:
-; CHECK:    ldbfminnml h0, h0, [x0]
+; CHECK:    stbfminnml h0, [x0]
     %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
     ret void
 }
@@ -1347,7 +1347,7 @@ define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %
 
 define dso_local void @atomicrmw_fmin_float_aligned_monotonic_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_monotonic_to_store:
-; CHECK:    ldfminnm s0, s0, [x0]
+; CHECK:    stfminnm s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
     ret void
 }
@@ -1368,7 +1368,7 @@ define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %va
 
 define dso_local void @atomicrmw_fmin_float_aligned_release_to_store(ptr %ptr, float %value) {
 ; CHECK-LABEL: atomicrmw_fmin_float_aligned_release_to_store:
-; CHECK:    ldfminnml s0, s0, [x0]
+; CHECK:    stfminnml s0, [x0]
     %r = atomicrmw fmin ptr %ptr, float %value release, align 4
     ret void
 }
@@ -1396,7 +1396,7 @@ define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, doubl
 
 define dso_local void @atomicrmw_fmin_double_aligned_monotonic_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_monotonic_to_store:
-; CHECK:    ldfminnm d0, d0, [x0]
+; CHECK:    stfminnm d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
     ret void
 }
@@ -1417,7 +1417,7 @@ define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double
 
 define dso_local void @atomicrmw_fmin_double_aligned_release_to_store(ptr %ptr, double %value) {
 ; CHECK-LABEL: atomicrmw_fmin_double_aligned_release_to_store:
-; CHECK:    ldfminnml d0, d0, [x0]
+; CHECK:    stfminnml d0, [x0]
     %r = atomicrmw fmin ptr %ptr, double %value release, align 8
     ret void
 }

>From d92da0a99bc0ac4b42bf8f21b242085fc60fbebb Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin at arm.com>
Date: Thu, 13 Mar 2025 17:45:59 +0000
Subject: [PATCH 3/3] - Fix formatting issues in generate-tests.py

---
 .../CodeGen/AArch64/Atomics/generate-tests.py    | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index b995c38463034..b4aebea8ff547 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -190,11 +190,11 @@ def relpath():
 
 
 def generate_store_opt_test(featname, ordering, op, alignval):
-  if featname != 'lsfe' or op == 'fsub' or alignval == 1:
-    return False
-  if ordering not in [AtomicOrder.monotonic, AtomicOrder.release]:
-    return False
-  return True;
+    if featname != "lsfe" or op == "fsub" or alignval == 1:
+        return False
+    if ordering not in [AtomicOrder.monotonic, AtomicOrder.release]:
+        return False
+    return True
 
 
 def align(val, aligned: bool) -> int:
@@ -222,14 +222,14 @@ def all_atomicrmw(f, datatype, atomicrmw_ops, featname):
                     if generate_store_opt_test(featname, ordering, op, alignval):
                         name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}_to_store"
                         f.write(
-                           textwrap.dedent(
-                               f"""
+                            textwrap.dedent(
+                                f"""
                            define dso_local void @{name}(ptr %ptr, {ty} %value) {{
                                %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval}
                                ret void
                            }}
                         """
-                           )
+                            )
                         )
 
 



More information about the llvm-commits mailing list