[llvm] 036f543 - [AArch64] Pre-commit tests for #125686 (NFC) (#126643)

via llvm-commits llvm-commits at lists.llvm.org
Wed Feb 12 03:48:16 PST 2025


Author: Jonathan Thackray
Date: 2025-02-12T11:48:11Z
New Revision: 036f543952570831a8c1ab41ec3f29c5eeea3919

URL: https://github.com/llvm/llvm-project/commit/036f543952570831a8c1ab41ec3f29c5eeea3919
DIFF: https://github.com/llvm/llvm-project/commit/036f543952570831a8c1ab41ec3f29c5eeea3919.diff

LOG: [AArch64] Pre-commit tests for #125686 (NFC) (#126643)

Update the `generate-tests.py` script to create new tests for `atomicrmw
{fadd,fmin,fmax}` and test these with `half`, `float`, `bfloat` and
`double`.

Generate fp auto-tests to check both with and without `+lsfe`, so that when
#125686 is merged, `+lsfe` will use a single atomic floating-point
instruction.

Added: 
    llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
    llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
    llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
    llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll

Modified: 
    llvm/test/CodeGen/AArch64/Atomics/generate-tests.py

Removed: 
    


################################################################################
diff  --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
new file mode 100644
index 0000000000000..992d050cf9ca0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -0,0 +1,1924 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}

diff  --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
new file mode 100644
index 0000000000000..b9dccdeeb600d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
@@ -0,0 +1,1924 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8a -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8a -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0:    ldaxrh w0, [x9]
+; -O0:    cmp w0, w10, uxth
+; -O0:    stlxrh w8, w11, [x9]
+; -O0:    subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0:    ldaxr w0, [x9]
+; -O0:    cmp w0, w10
+; -O0:    stlxr w8, w11, [x9]
+; -O0:    subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0:    ldaxr x0, [x9]
+; -O0:    cmp x0, x10
+; -O0:    stlxr w8, x11, [x9]
+; -O0:    subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}

diff  --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
new file mode 100644
index 0000000000000..6c46407177297
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -0,0 +1,1984 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}

diff  --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
new file mode 100644
index 0000000000000..bdd488e6933e5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
@@ -0,0 +1,1984 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+v8a -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+v8a -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1:    ldxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1:    ldaxrh w8, [x0]
+; -O1:    stlxrh w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1:    ldxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    ldaxrh w9, [x11]
+; -O0:    cmp w9, w8, uxth
+; -O0:    stlxrh w10, w12, [x11]
+; -O0:    subs w8, w9, w8, uxth
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1:    ldaxrh w9, [x0]
+; -O1:    add w9, w9, w8
+; -O1:    add w9, w10, w9
+; -O1:    stlxrh w10, w9, [x0]
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1:    ldxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1:    ldxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0:    ldaxr w9, [x11]
+; -O0:    cmp w9, w8
+; -O0:    stlxr w10, w12, [x11]
+; -O0:    subs w8, w9, w8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1:    ldaxr w8, [x0]
+; -O1:    stlxr w9, w8, [x0]
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1:    ldxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1:    ldxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0:    ldaxr x9, [x11]
+; -O0:    cmp x9, x8
+; -O0:    stlxr w10, x12, [x11]
+; -O0:    subs x8, x9, x8
+; -O0:    subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1:    ldaxr x8, [x0]
+; -O1:    stlxr w9, x8, [x0]
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+    ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+    ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+    ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+    ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0:    add w8, w8, w9
+; -O0:    add w8, w8, w9
+; -O0:    bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1:    add w8, w8, w20
+; -O1:    add w8, w9, w8
+; -O1:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+    ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+    ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+    ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+    ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK:    bl __atomic_compare_exchange
+    %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+    ret double %r
+}

diff  --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index ecda5fd69ca5d..f40bbaeb930c0 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -2,6 +2,7 @@
 import textwrap
 import enum
 import os
+import re
 
 """
 Generate the tests in llvm/test/CodeGen/AArch64/Atomics. Run from top level llvm-project.
@@ -13,20 +14,31 @@
 ]
 
 
-# Type name size
-class Type(enum.Enum):
-    # Value is the size in bytes
-    i8 = 1
-    i16 = 2
-    i32 = 4
-    i64 = 8
-    i128 = 16
+class ByteSizes:
+    def __init__(self, pairs):
+        if not isinstance(pairs, list):
+            raise ValueError("Must init with a list of key-value pairs")
 
-    def align(self, aligned: bool) -> int:
-        return self.value if aligned else 1
+        self._data = pairs[:]
+
+    def __iter__(self):
+        return iter(self._data)
 
-    def __str__(self) -> str:
-        return self.name
+
+# fmt: off
+Type = ByteSizes([
+   ("i8",   1),
+   ("i16",  2),
+   ("i32",  4),
+   ("i64",  8),
+   ("i128", 16)])
+
+FPType = ByteSizes([
+   ("half",   2),
+   ("bfloat", 2),
+   ("float",  4),
+   ("double", 8)])
+# fmt: on
 
 
 # Is this an aligned or unaligned access?
@@ -115,6 +127,9 @@ class Feature(enum.Flag):
     rcpc3 = enum.auto()  # FEAT_LSE2 + FEAT_LRCPC3
     lse2_lse128 = enum.auto()  # FEAT_LSE2 + FEAT_LSE128
 
+    def test_scope():
+        return "all"
+
     @property
     def mattr(self):
         if self == Feature.outline_atomics:
@@ -128,6 +143,21 @@ def mattr(self):
         return "+" + self.name
 
 
+class FPFeature(enum.Flag):
+    # Feature names in filenames are determined by the spelling here:
+    v8a_fp = enum.auto()
+    lsfe = enum.auto()  # FEAT_LSFE
+
+    def test_scope():
+        return "atomicrmw"
+
+    @property
+    def mattr(self):
+        if self == FPFeature.v8a_fp:
+            return "+v8a"
+        return "+" + self.name
+
+
 ATOMICRMW_OPS = [
     "xchg",
     "add",
@@ -142,11 +172,32 @@ def mattr(self):
     "umin",
 ]
 
+FP_ATOMICRMW_OPS = [
+    "fadd",
+    "fsub",
+    "fmax",
+    "fmin",
+]
+
+
+def relpath():
+    # __file__ changed to return absolute path in Python 3.9. Print only
+    # up to llvm-project (6 levels higher), to avoid unnecessary 
diff s and
+    # revealing directory structure of people running this script
+    top = "../" * 6
+    fp = os.path.relpath(__file__, os.path.abspath(os.path.join(__file__, top)))
+    return fp
+
 
-def all_atomicrmw(f):
-    for op in ATOMICRMW_OPS:
+def align(val, aligned: bool) -> int:
+    return val if aligned else 1
+
+
+def all_atomicrmw(f, datatype, atomicrmw_ops):
+    for op in atomicrmw_ops:
         for aligned in Aligned:
-            for ty in Type:
+            for ty, val in datatype:
+                alignval = align(val, aligned)
                 for ordering in ATOMICRMW_ORDERS:
                     name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}"
                     instr = "atomicrmw"
@@ -154,7 +205,7 @@ def all_atomicrmw(f):
                         textwrap.dedent(
                             f"""
                         define dso_local {ty} @{name}(ptr %ptr, {ty} %value) {{
-                            %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {ty.align(aligned)}
+                            %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval}
                             ret {ty} %r
                         }}
                     """
@@ -164,7 +215,8 @@ def all_atomicrmw(f):
 
 def all_load(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for ordering in ATOMIC_LOAD_ORDERS:
                 for const in [False, True]:
                     name = f"load_atomic_{ty}_{aligned}_{ordering}"
@@ -176,7 +228,7 @@ def all_load(f):
                         textwrap.dedent(
                             f"""
                         define dso_local {ty} @{name}({arg}) {{
-                            %r = {instr} {ty}, ptr %ptr {ordering}, align {ty.align(aligned)}
+                            %r = {instr} {ty}, ptr %ptr {ordering}, align {alignval}
                             ret {ty} %r
                         }}
                     """
@@ -186,7 +238,8 @@ def all_load(f):
 
 def all_store(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for ordering in ATOMIC_STORE_ORDERS:  # FIXME stores
                 name = f"store_atomic_{ty}_{aligned}_{ordering}"
                 instr = "store atomic"
@@ -194,7 +247,7 @@ def all_store(f):
                     textwrap.dedent(
                         f"""
                     define dso_local void @{name}({ty} %value, ptr %ptr) {{
-                        {instr} {ty} %value, ptr %ptr {ordering}, align {ty.align(aligned)}
+                        {instr} {ty} %value, ptr %ptr {ordering}, align {alignval}
                         ret void
                     }}
                 """
@@ -204,7 +257,8 @@ def all_store(f):
 
 def all_cmpxchg(f):
     for aligned in Aligned:
-        for ty in Type:
+        for ty, val in Type:
+            alignval = align(val, aligned)
             for success_ordering in CMPXCHG_SUCCESS_ORDERS:
                 for failure_ordering in CMPXCHG_FAILURE_ORDERS:
                     for weak in [False, True]:
@@ -217,7 +271,7 @@ def all_cmpxchg(f):
                             textwrap.dedent(
                                 f"""
                             define dso_local {ty} @{name}({ty} %expected, {ty} %new, ptr %ptr) {{
-                                %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {ty.align(aligned)}
+                                %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {alignval}
                                 %r = extractvalue {{ {ty}, i1 }} %pair, 0
                                 ret {ty} %r
                             }}
@@ -248,7 +302,8 @@ def header(f, triple, features, filter_args: str):
     )
     f.write(filter_args)
     f.write("\n")
-    f.write(f"; The base test file was generated by {__file__}\n")
+    f.write(f"; The base test file was generated by ./{relpath()}\n")
+
     for feat in features:
         for OptFlag in ["-O0", "-O1"]:
             f.write(
@@ -273,8 +328,7 @@ def header(f, triple, features, filter_args: str):
             )
 
 
-def write_lit_tests():
-    os.chdir("llvm/test/CodeGen/AArch64/Atomics/")
+def write_lit_tests(feature, datatypes, ops):
     for triple in TRIPLES:
         # Feature has no effect on fence, so keep it to one file.
         with open(f"{triple}-fence.ll", "w") as f:
@@ -282,11 +336,15 @@ def write_lit_tests():
             header(f, triple, Feature, filter_args)
             all_fence(f)
 
-        for feat in Feature:
+        for feat in feature:
             with open(f"{triple}-atomicrmw-{feat.name}.ll", "w") as f:
                 filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
                 header(f, triple, [feat], filter_args)
-                all_atomicrmw(f)
+                all_atomicrmw(f, datatypes, ops)
+
+            # Floating point atomics only supported for atomicrmw currently
+            if feature.test_scope() == "atomicrmw":
+                continue
 
             with open(f"{triple}-cmpxchg-{feat.name}.ll", "w") as f:
                 filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
@@ -305,7 +363,9 @@ def write_lit_tests():
 
 
 if __name__ == "__main__":
-    write_lit_tests()
+    os.chdir("llvm/test/CodeGen/AArch64/Atomics/")
+    write_lit_tests(Feature, Type, ATOMICRMW_OPS)
+    write_lit_tests(FPFeature, FPType, FP_ATOMICRMW_OPS)
 
     print(
         textwrap.dedent(


        


More information about the llvm-commits mailing list