[llvm] 036f543 - [AArch64] Pre-commit tests for #125686 (NFC) (#126643)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 12 03:48:16 PST 2025
Author: Jonathan Thackray
Date: 2025-02-12T11:48:11Z
New Revision: 036f543952570831a8c1ab41ec3f29c5eeea3919
URL: https://github.com/llvm/llvm-project/commit/036f543952570831a8c1ab41ec3f29c5eeea3919
DIFF: https://github.com/llvm/llvm-project/commit/036f543952570831a8c1ab41ec3f29c5eeea3919.diff
LOG: [AArch64] Pre-commit tests for #125686 (NFC) (#126643)
Update the `generate-tests.py` script to create new tests for `atomicrmw
{fadd,fmin,fmax}` and test these with `half`, `float`, `bfloat` and
`double`.
Generate fp auto-tests to check both with and without `+lsfe`, so that when
#125686 is merged, `+lsfe` will use a single atomic floating-point
instruction.
Added:
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
Modified:
llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
new file mode 100644
index 0000000000000..992d050cf9ca0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -0,0 +1,1924 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
new file mode 100644
index 0000000000000..b9dccdeeb600d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
@@ -0,0 +1,1924 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8a -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8a -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0: ldaxrh w0, [x9]
+; -O0: cmp w0, w10, uxth
+; -O0: stlxrh w8, w11, [x9]
+; -O0: subs w8, w8, w0, uxth
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0: ldaxr w0, [x9]
+; -O0: cmp w0, w10
+; -O0: stlxr w8, w11, [x9]
+; -O0: subs w8, w0, w8
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0: ldaxr x0, [x9]
+; -O0: cmp x0, x10
+; -O0: stlxr w8, x11, [x9]
+; -O0: subs x8, x0, x8
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
new file mode 100644
index 0000000000000..6c46407177297
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-lsfe.ll
@@ -0,0 +1,1984 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+lsfe -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
new file mode 100644
index 0000000000000..bdd488e6933e5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64_be-atomicrmw-v8a_fp.ll
@@ -0,0 +1,1984 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
+; The base test file was generated by ./llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+v8a -O0 | FileCheck %s --check-prefixes=CHECK,-O0
+; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64_be -mattr=+v8a -O1 | FileCheck %s --check-prefixes=CHECK,-O1
+
+define dso_local half @atomicrmw_fadd_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fadd_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fadd_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fadd_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fadd_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fadd_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fadd_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fadd ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fsub_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fsub_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fsub_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fsub_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fsub_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fsub_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fsub ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmax_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmax_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmax_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmax_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmax_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmax_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmax ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_monotonic(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_monotonic:
+; -O1: ldxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acquire(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acquire:
+; -O1: ldaxrh w8, [x0]
+; -O1: stxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_release(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_release:
+; -O1: ldxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_acq_rel(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_acq_rel:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 2
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value) {
+; -O0-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_half_aligned_seq_cst:
+; -O1: ldaxrh w8, [x0]
+; -O1: stlxrh w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 2
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
+; -O1: ldxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 2
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: ldaxrh w9, [x11]
+; -O0: cmp w9, w8, uxth
+; -O0: stlxrh w10, w12, [x11]
+; -O0: subs w8, w9, w8, uxth
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
+; -O1: ldaxrh w9, [x0]
+; -O1: add w9, w9, w8
+; -O1: add w9, w10, w9
+; -O1: stlxrh w10, w9, [x0]
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 2
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_monotonic(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_monotonic:
+; -O1: ldxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acquire(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acquire:
+; -O1: ldaxr w8, [x0]
+; -O1: stxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_release(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_release:
+; -O1: ldxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_acq_rel(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_acq_rel:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 4
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_aligned_seq_cst(ptr %ptr, float %value) {
+; -O0-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O0: ldaxr w9, [x11]
+; -O0: cmp w9, w8
+; -O0: stlxr w10, w12, [x11]
+; -O0: subs w8, w9, w8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_float_aligned_seq_cst:
+; -O1: ldaxr w8, [x0]
+; -O1: stlxr w9, w8, [x0]
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 4
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_monotonic(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_monotonic:
+; -O1: ldxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acquire(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acquire:
+; -O1: ldaxr x8, [x0]
+; -O1: stxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_release(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_release:
+; -O1: ldxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_acq_rel(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_acq_rel:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 8
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_aligned_seq_cst(ptr %ptr, double %value) {
+; -O0-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O0: ldaxr x9, [x11]
+; -O0: cmp x9, x8
+; -O0: stlxr w10, x12, [x11]
+; -O0: subs x8, x9, x8
+; -O0: subs w8, w8, #1
+;
+; -O1-LABEL: atomicrmw_fmin_double_aligned_seq_cst:
+; -O1: ldaxr x8, [x0]
+; -O1: stlxr w9, x8, [x0]
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 8
+ ret double %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_monotonic(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value monotonic, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acquire(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acquire, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_release(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value release, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_acq_rel(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value acq_rel, align 1
+ ret half %r
+}
+
+define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %value) {
+; CHECK-LABEL: atomicrmw_fmin_half_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, half %value seq_cst, align 1
+ ret half %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value monotonic, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acquire, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value release, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value acq_rel, align 1
+ ret bfloat %r
+}
+
+define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
+; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O0: add w8, w8, w9
+; -O0: add w8, w8, w9
+; -O0: bl __atomic_compare_exchange
+;
+; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
+; -O1: add w8, w8, w20
+; -O1: add w8, w9, w8
+; -O1: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, bfloat %value seq_cst, align 1
+ ret bfloat %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_monotonic(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value monotonic, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acquire(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acquire, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_release(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value release, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_acq_rel(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value acq_rel, align 1
+ ret float %r
+}
+
+define dso_local float @atomicrmw_fmin_float_unaligned_seq_cst(ptr %ptr, float %value) {
+; CHECK-LABEL: atomicrmw_fmin_float_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, float %value seq_cst, align 1
+ ret float %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_monotonic(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_monotonic:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value monotonic, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acquire(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acquire:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acquire, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_release(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_release:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value release, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_acq_rel(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_acq_rel:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value acq_rel, align 1
+ ret double %r
+}
+
+define dso_local double @atomicrmw_fmin_double_unaligned_seq_cst(ptr %ptr, double %value) {
+; CHECK-LABEL: atomicrmw_fmin_double_unaligned_seq_cst:
+; CHECK: bl __atomic_compare_exchange
+ %r = atomicrmw fmin ptr %ptr, double %value seq_cst, align 1
+ ret double %r
+}
diff --git a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
index ecda5fd69ca5d..f40bbaeb930c0 100755
--- a/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
+++ b/llvm/test/CodeGen/AArch64/Atomics/generate-tests.py
@@ -2,6 +2,7 @@
import textwrap
import enum
import os
+import re
"""
Generate the tests in llvm/test/CodeGen/AArch64/Atomics. Run from top level llvm-project.
@@ -13,20 +14,31 @@
]
-# Type name size
-class Type(enum.Enum):
- # Value is the size in bytes
- i8 = 1
- i16 = 2
- i32 = 4
- i64 = 8
- i128 = 16
+class ByteSizes:
+ def __init__(self, pairs):
+ if not isinstance(pairs, list):
+ raise ValueError("Must init with a list of key-value pairs")
- def align(self, aligned: bool) -> int:
- return self.value if aligned else 1
+ self._data = pairs[:]
+
+ def __iter__(self):
+ return iter(self._data)
- def __str__(self) -> str:
- return self.name
+
+# fmt: off
+Type = ByteSizes([
+ ("i8", 1),
+ ("i16", 2),
+ ("i32", 4),
+ ("i64", 8),
+ ("i128", 16)])
+
+FPType = ByteSizes([
+ ("half", 2),
+ ("bfloat", 2),
+ ("float", 4),
+ ("double", 8)])
+# fmt: on
# Is this an aligned or unaligned access?
@@ -115,6 +127,9 @@ class Feature(enum.Flag):
rcpc3 = enum.auto() # FEAT_LSE2 + FEAT_LRCPC3
lse2_lse128 = enum.auto() # FEAT_LSE2 + FEAT_LSE128
+ def test_scope():
+ return "all"
+
@property
def mattr(self):
if self == Feature.outline_atomics:
@@ -128,6 +143,21 @@ def mattr(self):
return "+" + self.name
+class FPFeature(enum.Flag):
+ # Feature names in filenames are determined by the spelling here:
+ v8a_fp = enum.auto()
+ lsfe = enum.auto() # FEAT_LSFE
+
+ def test_scope():
+ return "atomicrmw"
+
+ @property
+ def mattr(self):
+ if self == FPFeature.v8a_fp:
+ return "+v8a"
+ return "+" + self.name
+
+
ATOMICRMW_OPS = [
"xchg",
"add",
@@ -142,11 +172,32 @@ def mattr(self):
"umin",
]
+FP_ATOMICRMW_OPS = [
+ "fadd",
+ "fsub",
+ "fmax",
+ "fmin",
+]
+
+
+def relpath():
+ # __file__ changed to return absolute path in Python 3.9. Print only
+ # up to llvm-project (6 levels higher), to avoid unnecessary
diff s and
+ # revealing directory structure of people running this script
+ top = "../" * 6
+ fp = os.path.relpath(__file__, os.path.abspath(os.path.join(__file__, top)))
+ return fp
+
-def all_atomicrmw(f):
- for op in ATOMICRMW_OPS:
+def align(val, aligned: bool) -> int:
+ return val if aligned else 1
+
+
+def all_atomicrmw(f, datatype, atomicrmw_ops):
+ for op in atomicrmw_ops:
for aligned in Aligned:
- for ty in Type:
+ for ty, val in datatype:
+ alignval = align(val, aligned)
for ordering in ATOMICRMW_ORDERS:
name = f"atomicrmw_{op}_{ty}_{aligned}_{ordering}"
instr = "atomicrmw"
@@ -154,7 +205,7 @@ def all_atomicrmw(f):
textwrap.dedent(
f"""
define dso_local {ty} @{name}(ptr %ptr, {ty} %value) {{
- %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {ty.align(aligned)}
+ %r = {instr} {op} ptr %ptr, {ty} %value {ordering}, align {alignval}
ret {ty} %r
}}
"""
@@ -164,7 +215,8 @@ def all_atomicrmw(f):
def all_load(f):
for aligned in Aligned:
- for ty in Type:
+ for ty, val in Type:
+ alignval = align(val, aligned)
for ordering in ATOMIC_LOAD_ORDERS:
for const in [False, True]:
name = f"load_atomic_{ty}_{aligned}_{ordering}"
@@ -176,7 +228,7 @@ def all_load(f):
textwrap.dedent(
f"""
define dso_local {ty} @{name}({arg}) {{
- %r = {instr} {ty}, ptr %ptr {ordering}, align {ty.align(aligned)}
+ %r = {instr} {ty}, ptr %ptr {ordering}, align {alignval}
ret {ty} %r
}}
"""
@@ -186,7 +238,8 @@ def all_load(f):
def all_store(f):
for aligned in Aligned:
- for ty in Type:
+ for ty, val in Type:
+ alignval = align(val, aligned)
for ordering in ATOMIC_STORE_ORDERS: # FIXME stores
name = f"store_atomic_{ty}_{aligned}_{ordering}"
instr = "store atomic"
@@ -194,7 +247,7 @@ def all_store(f):
textwrap.dedent(
f"""
define dso_local void @{name}({ty} %value, ptr %ptr) {{
- {instr} {ty} %value, ptr %ptr {ordering}, align {ty.align(aligned)}
+ {instr} {ty} %value, ptr %ptr {ordering}, align {alignval}
ret void
}}
"""
@@ -204,7 +257,8 @@ def all_store(f):
def all_cmpxchg(f):
for aligned in Aligned:
- for ty in Type:
+ for ty, val in Type:
+ alignval = align(val, aligned)
for success_ordering in CMPXCHG_SUCCESS_ORDERS:
for failure_ordering in CMPXCHG_FAILURE_ORDERS:
for weak in [False, True]:
@@ -217,7 +271,7 @@ def all_cmpxchg(f):
textwrap.dedent(
f"""
define dso_local {ty} @{name}({ty} %expected, {ty} %new, ptr %ptr) {{
- %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {ty.align(aligned)}
+ %pair = {instr} ptr %ptr, {ty} %expected, {ty} %new {success_ordering} {failure_ordering}, align {alignval}
%r = extractvalue {{ {ty}, i1 }} %pair, 0
ret {ty} %r
}}
@@ -248,7 +302,8 @@ def header(f, triple, features, filter_args: str):
)
f.write(filter_args)
f.write("\n")
- f.write(f"; The base test file was generated by {__file__}\n")
+ f.write(f"; The base test file was generated by ./{relpath()}\n")
+
for feat in features:
for OptFlag in ["-O0", "-O1"]:
f.write(
@@ -273,8 +328,7 @@ def header(f, triple, features, filter_args: str):
)
-def write_lit_tests():
- os.chdir("llvm/test/CodeGen/AArch64/Atomics/")
+def write_lit_tests(feature, datatypes, ops):
for triple in TRIPLES:
# Feature has no effect on fence, so keep it to one file.
with open(f"{triple}-fence.ll", "w") as f:
@@ -282,11 +336,15 @@ def write_lit_tests():
header(f, triple, Feature, filter_args)
all_fence(f)
- for feat in Feature:
+ for feat in feature:
with open(f"{triple}-atomicrmw-{feat.name}.ll", "w") as f:
filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
header(f, triple, [feat], filter_args)
- all_atomicrmw(f)
+ all_atomicrmw(f, datatypes, ops)
+
+ # Floating point atomics only supported for atomicrmw currently
+ if feature.test_scope() == "atomicrmw":
+ continue
with open(f"{triple}-cmpxchg-{feat.name}.ll", "w") as f:
filter_args = r'--filter-out "\b(sp)\b" --filter "^\s*(ld[^r]|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"'
@@ -305,7 +363,9 @@ def write_lit_tests():
if __name__ == "__main__":
- write_lit_tests()
+ os.chdir("llvm/test/CodeGen/AArch64/Atomics/")
+ write_lit_tests(Feature, Type, ATOMICRMW_OPS)
+ write_lit_tests(FPFeature, FPType, FP_ATOMICRMW_OPS)
print(
textwrap.dedent(
More information about the llvm-commits
mailing list