[llvm] [AArch64][GlobalISel] Enable BF16 legalization for fadd and friends. (PR #196081)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 23:57:41 PDT 2026
https://github.com/davemgreen updated https://github.com/llvm/llvm-project/pull/196081
>From 93df729c9b0beaa0d41a7a355bd5389c691d79e3 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Sat, 9 May 2026 07:57:26 +0100
Subject: [PATCH] [AArch64][GlobalISel] Enable BF16 legalization for fadd and
friends.
This enabled bf16 promotion for the following operations in GISel, promoting
them to f32 and truncating the result back:
G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +-
.../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 285 +-
.../Atomics/aarch64-atomicrmw-v8a_fp.ll | 660 ++--
.../GlobalISel/legalizer-info-validation.mir | 68 +-
.../test/CodeGen/AArch64/bf16-instructions.ll | 1302 ++++---
.../CodeGen/AArch64/bf16-v4-instructions.ll | 1558 ++++++---
.../CodeGen/AArch64/bf16-v8-instructions.ll | 3108 +++++++++++------
7 files changed, 4564 insertions(+), 2424 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 605cb86a7bb60..88d2455aae425 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -450,7 +450,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor(HasFP16, {f16, v4f16, v8f16})
.libcallFor({f128})
.scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
- .minScalarOrElt(0, MinFPScalar)
+ .widenScalarIf(
+ [=](const LegalityQuery &Q) {
+ return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
+ Q.Types[0].getScalarType().isBFloat16();
+ },
+ changeElementTo(0, f32))
.clampNumElements(0, v4s16, v8s16)
.clampNumElements(0, v2s32, v4s32)
.clampNumElements(0, v2s64, v2s64)
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 290c8a60d700d..6a0b689b16888 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -239,8 +239,11 @@ define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
@@ -253,8 +256,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
@@ -267,8 +273,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
@@ -281,8 +290,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
@@ -295,8 +307,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
@@ -449,12 +464,15 @@ define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -467,12 +485,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -485,12 +506,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -503,12 +527,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -521,12 +548,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -714,8 +744,11 @@ define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
@@ -728,8 +761,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
@@ -742,8 +778,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
@@ -756,8 +795,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
@@ -770,8 +812,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
@@ -1085,8 +1130,11 @@ define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
@@ -1099,8 +1147,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
@@ -1113,8 +1164,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
@@ -1127,8 +1181,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
@@ -1141,8 +1198,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
@@ -1456,8 +1516,11 @@ define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
@@ -1470,8 +1533,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
@@ -1484,8 +1550,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
@@ -1498,8 +1567,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
@@ -1512,8 +1584,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
@@ -1827,8 +1902,11 @@ define dso_local half @atomicrmw_fmaximum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
@@ -1841,8 +1919,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
@@ -1855,8 +1936,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
@@ -1869,8 +1953,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
@@ -1883,8 +1970,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
@@ -2198,8 +2288,11 @@ define dso_local half @atomicrmw_fminimum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
@@ -2212,8 +2305,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
@@ -2226,8 +2322,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
@@ -2240,8 +2339,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
@@ -2254,8 +2356,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
index ccb4fd61b002c..d71231059c3b3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
@@ -75,12 +75,15 @@ define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -93,12 +96,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -111,12 +117,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -129,12 +138,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -147,12 +159,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -340,8 +355,11 @@ define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
@@ -354,8 +372,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
@@ -368,8 +389,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
@@ -382,8 +406,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
@@ -396,8 +423,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
@@ -550,12 +580,15 @@ define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -568,12 +601,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -586,12 +622,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -604,12 +643,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -622,12 +664,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -815,8 +860,11 @@ define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
@@ -829,8 +877,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
@@ -843,8 +894,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
@@ -857,8 +911,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
@@ -871,8 +928,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
@@ -1025,12 +1085,15 @@ define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1043,12 +1106,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -1061,12 +1127,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -1079,12 +1148,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -1097,12 +1169,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -1290,8 +1365,11 @@ define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
@@ -1304,8 +1382,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
@@ -1318,8 +1399,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
@@ -1332,8 +1416,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
@@ -1346,8 +1433,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
@@ -1500,12 +1590,15 @@ define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1518,12 +1611,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -1536,12 +1632,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -1554,12 +1653,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -1572,12 +1674,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -1765,8 +1870,11 @@ define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
@@ -1779,8 +1887,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
@@ -1793,8 +1904,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
@@ -1807,8 +1921,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
@@ -1821,8 +1938,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
@@ -1975,12 +2095,15 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_seq_cst(ptr %ptr, half %v
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1993,12 +2116,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -2011,12 +2137,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -2029,12 +2158,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -2047,12 +2179,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -2240,8 +2375,11 @@ define dso_local half @atomicrmw_fmaximum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
@@ -2254,8 +2392,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
@@ -2268,8 +2409,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
@@ -2282,8 +2426,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
@@ -2296,8 +2443,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
@@ -2450,12 +2600,15 @@ define dso_local half @atomicrmw_fminimum_half_aligned_seq_cst(ptr %ptr, half %v
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -2468,12 +2621,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -2486,12 +2642,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -2504,12 +2663,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -2522,12 +2684,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -2715,8 +2880,11 @@ define dso_local half @atomicrmw_fminimum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
@@ -2729,8 +2897,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
@@ -2743,8 +2914,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
@@ -2757,8 +2931,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
@@ -2771,8 +2948,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index ff4fbc0e1afeb..17bb373070568 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -179,12 +179,12 @@
#
# DEBUG-NEXT: G_INTRINSIC_TRUNC (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_ROUND (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
@@ -195,8 +195,8 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_READCYCLECOUNTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -516,27 +516,27 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FADD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FSUB (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMUL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMA (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FDIV (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FREM (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -627,12 +627,12 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMINNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAXNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMINNUM_IEEE (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -641,12 +641,12 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMINIMUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAXIMUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMINIMUMNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -772,8 +772,8 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FCEIL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FCOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -818,20 +818,20 @@
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FFLOOR (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FRINT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FNEARBYINT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_ADDRSPACE_CAST (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index d9f661d2d1d34..05bf7a3edbd45 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -4,12 +4,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc_f32_f16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
@@ -39,7 +34,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -57,26 +51,10 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
-;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+;
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc_f32_f16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
@@ -106,7 +84,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32_fadd
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -124,184 +101,341 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
define bfloat @test_fadd(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fadd bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fsub(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fsub:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsub s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fsub:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsub s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fsub:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fsub s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fsub:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fsub s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fsub:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fsub s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fsub:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fsub s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fsub bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fmul(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fmul:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmul:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmul:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmul:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmul:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmul:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fmul bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fmadd(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fmadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: fmadd s0, s0, s1, s2
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %mul = fmul fast bfloat %a, %b
- %r = fadd fast bfloat %mul, %c
- ret bfloat %r
-}
-
-define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fdiv:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fdiv s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fdiv:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fdiv s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-CVT-GI-LABEL: test_fmadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w11, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %mul = fmul fast bfloat %a, %b
+ %r = fadd fast bfloat %mul, %c
+ ret bfloat %r
+}
+
+define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fdiv:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fdiv s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_fdiv:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fdiv s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fdiv:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fdiv s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fdiv:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fdiv s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fdiv bfloat %a, %b
ret bfloat %r
}
@@ -1212,20 +1346,20 @@ define bfloat @test_bitcast_f16tobfloat(half %a) #0 {
}
define bfloat @test_sqrt(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_sqrt:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsqrt s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sqrt:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsqrt s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sqrt:
; CHECK-BF16: // %bb.0:
@@ -1234,6 +1368,24 @@ define bfloat @test_sqrt(bfloat %a) #0 {
; CHECK-BF16-NEXT: fsqrt s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sqrt:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fsqrt s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.sqrt.bf16(bfloat %a)
ret bfloat %r
}
@@ -1812,36 +1964,70 @@ define bfloat @test_log2(bfloat %a) #0 {
}
define bfloat @test_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fma:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmadd s0, s0, s1, s2
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fma:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fma:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmadd s0, s0, s1, s2
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fma:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fma:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmadd s0, s0, s1, s2
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fma:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %r
}
@@ -1873,107 +2059,167 @@ define bfloat @test_fneg(bfloat %a) #0 {
}
define bfloat @test_minnum(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_minnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fminnm s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
-;
-; CHECK-BF16-LABEL: test_minnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fminnm s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %r = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
- ret bfloat %r
-}
-
-define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_maxnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
-;
-; CHECK-BF16-LABEL: test_maxnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
- ret bfloat %r
-}
-
-define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-SD-LABEL: test_copysign:
+; CHECK-CVT-SD-LABEL: test_minnum:
; CHECK-CVT-SD: // %bb.0:
; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v2.16b
-; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: fminnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
; CHECK-CVT-SD-NEXT: fmov s0, w8
; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-SD-LABEL: test_copysign:
+; CHECK-BF16-SD-LABEL: test_minnum:
; CHECK-BF16-SD: // %bb.0:
; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-SD-NEXT: mvni v2.4s, #128, lsl #24
; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-SD-NEXT: fminnm s0, s0, s1
; CHECK-BF16-SD-NEXT: bfcvt h0, s0
; CHECK-BF16-SD-NEXT: ret
;
-; CHECK-CVT-GI-LABEL: test_copysign:
+; CHECK-CVT-GI-LABEL: test_minnum:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: mvni v2.4h, #128, lsl #8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-GI-NEXT: bif v0.8b, v1.8b, v2.8b
-; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
-; CHECK-CVT-GI-NEXT: ret
-;
-; CHECK-BF16-GI-LABEL: test_copysign:
-; CHECK-BF16-GI: // %bb.0:
-; CHECK-BF16-GI-NEXT: mvni v2.4h, #128, lsl #8
-; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fminnm s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_minnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fminnm s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %r = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_maxnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_maxnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_maxnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmaxnm s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_maxnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_copysign:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_copysign:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: mvni v2.4h, #128, lsl #8
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
; CHECK-BF16-GI-NEXT: bif v0.8b, v1.8b, v2.8b
; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
; CHECK-BF16-GI-NEXT: ret
@@ -2147,20 +2393,20 @@ define float @test_copysign_extended(bfloat %a, bfloat %b) #0 {
}
define bfloat @test_floor(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_floor:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintm s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_floor:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintm s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_floor:
; CHECK-BF16: // %bb.0:
@@ -2169,25 +2415,43 @@ define bfloat @test_floor(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintm s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_floor:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintm s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.floor.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_ceil(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_ceil:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintp s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_ceil:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintp s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_ceil:
; CHECK-BF16: // %bb.0:
@@ -2196,25 +2460,43 @@ define bfloat @test_ceil(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintp s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_ceil:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintp s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.ceil.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_trunc(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_trunc:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintz s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_trunc:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintz s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_trunc:
; CHECK-BF16: // %bb.0:
@@ -2223,25 +2505,43 @@ define bfloat @test_trunc(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintz s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_trunc:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintz s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.trunc.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_rint(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_rint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintx s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_rint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintx s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_rint:
; CHECK-BF16: // %bb.0:
@@ -2250,25 +2550,43 @@ define bfloat @test_rint(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintx s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_rint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintx s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.rint.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_nearbyint(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_nearbyint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frinti s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_nearbyint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frinti s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_nearbyint:
; CHECK-BF16: // %bb.0:
@@ -2277,25 +2595,43 @@ define bfloat @test_nearbyint(bfloat %a) #0 {
; CHECK-BF16-NEXT: frinti s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_nearbyint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frinti s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.nearbyint.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_round(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_round:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frinta s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_round:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frinta s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_round:
; CHECK-BF16: // %bb.0:
@@ -2304,25 +2640,43 @@ define bfloat @test_round(bfloat %a) #0 {
; CHECK-BF16-NEXT: frinta s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_round:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frinta s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_roundeven(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_roundeven:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintn s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_roundeven:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintn s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_roundeven:
; CHECK-BF16: // %bb.0:
@@ -2331,52 +2685,118 @@ define bfloat @test_roundeven(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintn s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_roundeven:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintn s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.roundeven.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_fmuladd(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fmuladd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmuladd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmuladd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul s0, s0, s1
-; CHECK-BF16-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmuladd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul s0, s0, s1
+; CHECK-BF16-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmuladd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w11, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmuladd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = call bfloat @llvm.fmuladd.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %r
}
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index c10436c93614c..823a245a3cc81 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -4,12 +4,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ueq
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ugt
@@ -40,7 +35,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -58,26 +52,10 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
-;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+;
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ueq
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ugt
@@ -108,7 +86,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -126,19 +103,8 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
define <4 x bfloat> @test_build(<4 x bfloat> %a) {
; CHECK-CVT-SD-LABEL: test_build:
@@ -168,138 +134,289 @@ define <4 x bfloat> @test_build(<4 x bfloat> %a) {
}
define <4 x bfloat> @test_fadd(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_fsub(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fsub:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsub v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fsub:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fsub:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fsub v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fsub:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fsub:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fsub:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fsub <4 x bfloat> %a, %b
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_fmul(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fmul:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmul:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmul:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmul:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmul:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmul:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fmul <4 x bfloat> %a, %b
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_fmadd(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fmadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v4.4s
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v4.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v3.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v7.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v5.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v7.4s, v4.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%mul = fmul fast <4 x bfloat> %a, %b
%r = fadd fast <4 x bfloat> %mul, %c
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_fdiv(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fdiv:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fdiv v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fdiv:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fdiv:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fdiv v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fdiv:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fdiv:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fdiv:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fdiv <4 x bfloat> %a, %b
ret <4 x bfloat> %r
}
@@ -1154,74 +1271,99 @@ define <4 x bfloat> @test_bitcast_f16tobfloat(float, <4 x half> %a) {
}
define <4 x bfloat> @test_sqrt(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_sqrt:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT: mov h1, v0.h[1]
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: mov h3, v0.h[2]
-; CHECK-CVT-NEXT: mov h0, v0.h[3]
-; CHECK-CVT-NEXT: fsqrt s2, s2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmov w10, s2
-; CHECK-CVT-NEXT: ubfx w12, w10, #16, #1
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: fsqrt s1, s1
-; CHECK-CVT-NEXT: fmov w9, s1
-; CHECK-CVT-NEXT: ubfx w11, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fsqrt s3, s3
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: fmov w11, s3
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w9, w11, w8
-; CHECK-CVT-NEXT: add w9, w12, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fsqrt s4, s0
-; CHECK-CVT-NEXT: fmov s0, w10
-; CHECK-CVT-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sqrt:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT: mov h1, v0.h[1]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[3]
+; CHECK-CVT-SD-NEXT: fsqrt s2, s2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w10, s2
+; CHECK-CVT-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: fsqrt s1, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s1
+; CHECK-CVT-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fsqrt s3, s3
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: fmov w11, s3
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w11, w8
+; CHECK-CVT-SD-NEXT: add w9, w12, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fsqrt s4, s0
+; CHECK-CVT-SD-NEXT: fmov s0, w10
+; CHECK-CVT-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sqrt:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-BF16-NEXT: mov h1, v0.h[1]
-; CHECK-BF16-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h3, v0.h[2]
-; CHECK-BF16-NEXT: mov h0, v0.h[3]
-; CHECK-BF16-NEXT: fsqrt s2, s2
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fsqrt s1, s1
-; CHECK-BF16-NEXT: bfcvt h1, s1
-; CHECK-BF16-NEXT: fsqrt s3, s3
-; CHECK-BF16-NEXT: fsqrt s4, s0
-; CHECK-BF16-NEXT: bfcvt h0, s2
-; CHECK-BF16-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-BF16-NEXT: bfcvt h1, s3
-; CHECK-BF16-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-BF16-NEXT: bfcvt h1, s4
-; CHECK-BF16-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_sqrt:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT: mov h1, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h0, v0.h[3]
+; CHECK-BF16-SD-NEXT: fsqrt s2, s2
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fsqrt s1, s1
+; CHECK-BF16-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16-SD-NEXT: fsqrt s3, s3
+; CHECK-BF16-SD-NEXT: fsqrt s4, s0
+; CHECK-BF16-SD-NEXT: bfcvt h0, s2
+; CHECK-BF16-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h1, s3
+; CHECK-BF16-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h1, s4
+; CHECK-BF16-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sqrt:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsqrt v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sqrt:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fsqrt v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.sqrt.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
@@ -3078,106 +3220,135 @@ define <4 x bfloat> @test_log2(<4 x bfloat> %a) #0 {
}
define <4 x bfloat> @test_fma(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fma:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $d2 killed $d2 def $q2
-; CHECK-CVT-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT: mov h3, v2.h[1]
-; CHECK-CVT-NEXT: mov h4, v1.h[1]
-; CHECK-CVT-NEXT: mov w9, #32767 // =0x7fff
-; CHECK-CVT-NEXT: mov h5, v0.h[1]
-; CHECK-CVT-NEXT: shll v6.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v7.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v16.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h17, v2.h[2]
-; CHECK-CVT-NEXT: mov h18, v1.h[2]
-; CHECK-CVT-NEXT: mov h19, v0.h[2]
-; CHECK-CVT-NEXT: mov h2, v2.h[3]
-; CHECK-CVT-NEXT: mov h1, v1.h[3]
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-CVT-NEXT: mov h0, v0.h[3]
-; CHECK-CVT-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: fmadd s3, s5, s4, s3
-; CHECK-CVT-NEXT: fmadd s4, s16, s7, s6
-; CHECK-CVT-NEXT: shll v5.4s, v17.4h, #16
-; CHECK-CVT-NEXT: shll v6.4s, v18.4h, #16
-; CHECK-CVT-NEXT: shll v7.4s, v19.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmov w8, s3
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: fmadd s3, s7, s6, s5
-; CHECK-CVT-NEXT: fmadd s1, s0, s1, s2
-; CHECK-CVT-NEXT: ubfx w11, w8, #16, #1
-; CHECK-CVT-NEXT: ubfx w12, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w9
-; CHECK-CVT-NEXT: add w10, w10, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: fmov w11, s3
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w8
-; CHECK-CVT-NEXT: fmov s0, w10
-; CHECK-CVT-NEXT: add w8, w11, w9
-; CHECK-CVT-NEXT: fmov w10, s1
-; CHECK-CVT-NEXT: add w8, w12, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: add w8, w10, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fma:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-CVT-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT: mov h3, v2.h[1]
+; CHECK-CVT-SD-NEXT: mov h4, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov w9, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: mov h5, v0.h[1]
+; CHECK-CVT-SD-NEXT: shll v6.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h17, v2.h[2]
+; CHECK-CVT-SD-NEXT: mov h18, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h19, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h2, v2.h[3]
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[3]
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[3]
+; CHECK-CVT-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: fmadd s3, s5, s4, s3
+; CHECK-CVT-SD-NEXT: fmadd s4, s16, s7, s6
+; CHECK-CVT-SD-NEXT: shll v5.4s, v17.4h, #16
+; CHECK-CVT-SD-NEXT: shll v6.4s, v18.4h, #16
+; CHECK-CVT-SD-NEXT: shll v7.4s, v19.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w8, s3
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: fmadd s3, s7, s6, s5
+; CHECK-CVT-SD-NEXT: fmadd s1, s0, s1, s2
+; CHECK-CVT-SD-NEXT: ubfx w11, w8, #16, #1
+; CHECK-CVT-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w9
+; CHECK-CVT-SD-NEXT: add w10, w10, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: fmov w11, s3
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w8
+; CHECK-CVT-SD-NEXT: fmov s0, w10
+; CHECK-CVT-SD-NEXT: add w8, w11, w9
+; CHECK-CVT-SD-NEXT: fmov w10, s1
+; CHECK-CVT-SD-NEXT: add w8, w12, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fma:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $d2 killed $d2 def $q2
-; CHECK-BF16-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-BF16-NEXT: mov h3, v2.h[1]
-; CHECK-BF16-NEXT: mov h4, v1.h[1]
-; CHECK-BF16-NEXT: mov h5, v0.h[1]
-; CHECK-BF16-NEXT: shll v6.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v7.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h17, v2.h[2]
-; CHECK-BF16-NEXT: mov h18, v1.h[2]
-; CHECK-BF16-NEXT: mov h19, v0.h[2]
-; CHECK-BF16-NEXT: mov h2, v2.h[3]
-; CHECK-BF16-NEXT: mov h1, v1.h[3]
-; CHECK-BF16-NEXT: fmadd s6, s16, s7, s6
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: mov h16, v0.h[3]
-; CHECK-BF16-NEXT: shll v7.4s, v19.4h, #16
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fmadd s3, s5, s4, s3
-; CHECK-BF16-NEXT: shll v4.4s, v17.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v18.4h, #16
-; CHECK-BF16-NEXT: bfcvt h0, s6
-; CHECK-BF16-NEXT: fmadd s4, s7, s5, s4
-; CHECK-BF16-NEXT: shll v5.4s, v16.4h, #16
-; CHECK-BF16-NEXT: bfcvt h3, s3
-; CHECK-BF16-NEXT: fmadd s1, s5, s1, s2
-; CHECK-BF16-NEXT: mov v0.h[1], v3.h[0]
-; CHECK-BF16-NEXT: bfcvt h3, s4
-; CHECK-BF16-NEXT: bfcvt h1, s1
-; CHECK-BF16-NEXT: mov v0.h[2], v3.h[0]
-; CHECK-BF16-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fma:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-BF16-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT: mov h3, v2.h[1]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h17, v2.h[2]
+; CHECK-BF16-SD-NEXT: mov h18, v1.h[2]
+; CHECK-BF16-SD-NEXT: mov h19, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h2, v2.h[3]
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[3]
+; CHECK-BF16-SD-NEXT: fmadd s6, s16, s7, s6
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[3]
+; CHECK-BF16-SD-NEXT: shll v7.4s, v19.4h, #16
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s3, s5, s4, s3
+; CHECK-BF16-SD-NEXT: shll v4.4s, v17.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v18.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h0, s6
+; CHECK-BF16-SD-NEXT: fmadd s4, s7, s5, s4
+; CHECK-BF16-SD-NEXT: shll v5.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16-SD-NEXT: fmadd s1, s5, s1, s2
+; CHECK-BF16-SD-NEXT: mov v0.h[1], v3.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h3, s4
+; CHECK-BF16-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[2], v3.h[0]
+; CHECK-BF16-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fma:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: bit v0.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fma:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.fma.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %r
}
@@ -3202,179 +3373,233 @@ define <4 x bfloat> @test_fneg(<4 x bfloat> %a) #0 {
}
define <4 x bfloat> @test_minnum(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
-; CHECK-CVT-LABEL: test_minnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT: mov h2, v1.h[1]
-; CHECK-CVT-NEXT: mov h3, v0.h[1]
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h6, v1.h[2]
-; CHECK-CVT-NEXT: mov h7, v0.h[2]
-; CHECK-CVT-NEXT: mov h1, v1.h[3]
-; CHECK-CVT-NEXT: mov h0, v0.h[3]
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fminnm s2, s3, s2
-; CHECK-CVT-NEXT: fminnm s3, s5, s4
-; CHECK-CVT-NEXT: shll v4.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-CVT-NEXT: fminnm s1, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s2
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: fminnm s2, s5, s4
-; CHECK-CVT-NEXT: ubfx w11, w9, #16, #1
-; CHECK-CVT-NEXT: ubfx w12, w10, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: fmov w11, s2
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: fmov s0, w10
-; CHECK-CVT-NEXT: add w9, w11, w8
-; CHECK-CVT-NEXT: fmov w10, s1
-; CHECK-CVT-NEXT: add w9, w12, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_minnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT: mov h2, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[3]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[3]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fminnm s2, s3, s2
+; CHECK-CVT-SD-NEXT: fminnm s3, s5, s4
+; CHECK-CVT-SD-NEXT: shll v4.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: fminnm s1, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s2
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: fminnm s2, s5, s4
+; CHECK-CVT-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-CVT-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: fmov w11, s2
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: fmov s0, w10
+; CHECK-CVT-SD-NEXT: add w9, w11, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s1
+; CHECK-CVT-SD-NEXT: add w9, w12, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_minnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-BF16-NEXT: mov h2, v1.h[1]
-; CHECK-BF16-NEXT: mov h3, v0.h[1]
-; CHECK-BF16-NEXT: mov h4, v1.h[2]
-; CHECK-BF16-NEXT: shll v5.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v6.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h7, v0.h[2]
-; CHECK-BF16-NEXT: mov h1, v1.h[3]
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fminnm s2, s3, s2
-; CHECK-BF16-NEXT: fminnm s3, s6, s5
-; CHECK-BF16-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-BF16-NEXT: mov h6, v0.h[3]
-; CHECK-BF16-NEXT: fminnm s4, s5, s4
-; CHECK-BF16-NEXT: bfcvt h2, s2
-; CHECK-BF16-NEXT: bfcvt h0, s3
-; CHECK-BF16-NEXT: shll v3.4s, v6.4h, #16
-; CHECK-BF16-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-BF16-NEXT: bfcvt h2, s4
-; CHECK-BF16-NEXT: fminnm s1, s3, s1
-; CHECK-BF16-NEXT: mov v0.h[2], v2.h[0]
-; CHECK-BF16-NEXT: bfcvt h1, s1
-; CHECK-BF16-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_minnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[2]
+; CHECK-BF16-SD-NEXT: shll v5.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v6.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[3]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fminnm s2, s3, s2
+; CHECK-BF16-SD-NEXT: fminnm s3, s6, s5
+; CHECK-BF16-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: mov h6, v0.h[3]
+; CHECK-BF16-SD-NEXT: fminnm s4, s5, s4
+; CHECK-BF16-SD-NEXT: bfcvt h2, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s3
+; CHECK-BF16-SD-NEXT: shll v3.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16-SD-NEXT: fminnm s1, s3, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[2], v2.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_minnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fminnm v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_minnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fminnm v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.minnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_maxnum(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
-; CHECK-CVT-LABEL: test_maxnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT: mov h2, v1.h[1]
-; CHECK-CVT-NEXT: mov h3, v0.h[1]
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h6, v1.h[2]
-; CHECK-CVT-NEXT: mov h7, v0.h[2]
-; CHECK-CVT-NEXT: mov h1, v1.h[3]
-; CHECK-CVT-NEXT: mov h0, v0.h[3]
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s2, s3, s2
-; CHECK-CVT-NEXT: fmaxnm s3, s5, s4
-; CHECK-CVT-NEXT: shll v4.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s1, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s2
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: fmaxnm s2, s5, s4
-; CHECK-CVT-NEXT: ubfx w11, w9, #16, #1
-; CHECK-CVT-NEXT: ubfx w12, w10, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: fmov w11, s2
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: fmov s0, w10
-; CHECK-CVT-NEXT: add w9, w11, w8
-; CHECK-CVT-NEXT: fmov w10, s1
-; CHECK-CVT-NEXT: add w9, w12, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-CVT-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_maxnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT: mov h2, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[3]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[3]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s2, s3, s2
+; CHECK-CVT-SD-NEXT: fmaxnm s3, s5, s4
+; CHECK-CVT-SD-NEXT: shll v4.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s1, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s2
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: fmaxnm s2, s5, s4
+; CHECK-CVT-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-CVT-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: fmov w11, s2
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: fmov s0, w10
+; CHECK-CVT-SD-NEXT: add w9, w11, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s1
+; CHECK-CVT-SD-NEXT: add w9, w12, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-CVT-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_maxnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-BF16-NEXT: mov h2, v1.h[1]
-; CHECK-BF16-NEXT: mov h3, v0.h[1]
-; CHECK-BF16-NEXT: mov h4, v1.h[2]
-; CHECK-BF16-NEXT: shll v5.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v6.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h7, v0.h[2]
-; CHECK-BF16-NEXT: mov h1, v1.h[3]
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s2, s3, s2
-; CHECK-BF16-NEXT: fmaxnm s3, s6, s5
-; CHECK-BF16-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-BF16-NEXT: mov h6, v0.h[3]
-; CHECK-BF16-NEXT: fmaxnm s4, s5, s4
-; CHECK-BF16-NEXT: bfcvt h2, s2
-; CHECK-BF16-NEXT: bfcvt h0, s3
-; CHECK-BF16-NEXT: shll v3.4s, v6.4h, #16
-; CHECK-BF16-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-BF16-NEXT: bfcvt h2, s4
-; CHECK-BF16-NEXT: fmaxnm s1, s3, s1
-; CHECK-BF16-NEXT: mov v0.h[2], v2.h[0]
-; CHECK-BF16-NEXT: bfcvt h1, s1
-; CHECK-BF16-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-BF16-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_maxnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[2]
+; CHECK-BF16-SD-NEXT: shll v5.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v6.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[3]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s2, s3, s2
+; CHECK-BF16-SD-NEXT: fmaxnm s3, s6, s5
+; CHECK-BF16-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: mov h6, v0.h[3]
+; CHECK-BF16-SD-NEXT: fmaxnm s4, s5, s4
+; CHECK-BF16-SD-NEXT: bfcvt h2, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s3
+; CHECK-BF16-SD-NEXT: shll v3.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16-SD-NEXT: fmaxnm s1, s3, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[2], v2.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16-SD-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_maxnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_maxnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.maxnum.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b)
ret <4 x bfloat> %r
}
@@ -3436,17 +3661,17 @@ define <4 x bfloat> @test_copysign_f32(<4 x bfloat> %a, <4 x float> %b) #0 {
}
define <4 x bfloat> @test_floor(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_floor:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frintm v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_floor:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frintm v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_floor:
; CHECK-BF16: // %bb.0:
@@ -3454,22 +3679,40 @@ define <4 x bfloat> @test_floor(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frintm v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_floor:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintm v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.floor.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_ceil(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_ceil:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frintp v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_ceil:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frintp v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_ceil:
; CHECK-BF16: // %bb.0:
@@ -3477,22 +3720,40 @@ define <4 x bfloat> @test_ceil(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frintp v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_ceil:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintp v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.ceil.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_trunc(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_trunc:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frintz v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_trunc:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frintz v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_trunc:
; CHECK-BF16: // %bb.0:
@@ -3500,22 +3761,40 @@ define <4 x bfloat> @test_trunc(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frintz v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_trunc:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintz v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.trunc.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_rint(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_rint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frintx v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_rint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frintx v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_rint:
; CHECK-BF16: // %bb.0:
@@ -3523,22 +3802,40 @@ define <4 x bfloat> @test_rint(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frintx v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_rint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintx v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.rint.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_nearbyint(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_nearbyint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frinti v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_nearbyint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frinti v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_nearbyint:
; CHECK-BF16: // %bb.0:
@@ -3546,22 +3843,40 @@ define <4 x bfloat> @test_nearbyint(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frinti v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_nearbyint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frinti v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.nearbyint.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_round(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_round:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frinta v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_round:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frinta v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_round:
; CHECK-BF16: // %bb.0:
@@ -3569,22 +3884,40 @@ define <4 x bfloat> @test_round(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frinta v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_round:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frinta v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.round.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_roundeven(<4 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_roundeven:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: frintn v0.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_roundeven:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: frintn v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_roundeven:
; CHECK-BF16: // %bb.0:
@@ -3592,42 +3925,103 @@ define <4 x bfloat> @test_roundeven(<4 x bfloat> %a) #0 {
; CHECK-BF16-NEXT: frintn v0.4s, v0.4s
; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_roundeven:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintn v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.roundeven.v4bf16(<4 x bfloat> %a)
ret <4 x bfloat> %r
}
define <4 x bfloat> @test_fmuladd(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fmuladd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v4.4s
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v4.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmuladd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v3.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v4.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmuladd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmuladd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmuladd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v5.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v7.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v5.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v7.4s, v4.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmuladd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <4 x bfloat> @llvm.fmuladd.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
ret <4 x bfloat> %r
}
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 7c0b4092a6f48..2b31e876487d9 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -5,12 +5,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ueq
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ugt
@@ -45,7 +40,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -63,26 +57,10 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ueq
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ugt
@@ -117,7 +95,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i32
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_uitofp_i64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -135,19 +112,8 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fneg
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
define <8 x bfloat> @test_build(<8 x bfloat> %a) {
; CHECK-CVT-SD-LABEL: test_build:
@@ -183,97 +149,209 @@ define <8 x bfloat> @test_build(<8 x bfloat> %a) {
}
define <8 x bfloat> @test_fadd(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fadd v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fadd v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fadd v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fadd v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fadd v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fadd:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fadd v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fadd v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fadd v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fadd <8 x bfloat> %a, %b
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_fsub(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fsub:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fsub v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fsub v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fsub:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fsub v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fsub:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fsub v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fsub v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fsub:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fsub v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fsub:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fsub v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fsub:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsub v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fsub:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fsub v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fsub <8 x bfloat> %a, %b
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_fmul(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fmul:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fmul v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fmul v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmul:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-SD-LABEL: test_fmul:
; CHECK-BF16-SD: // %bb.0:
@@ -299,13 +377,45 @@ define <8 x bfloat> @test_fmul(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-BF16SVE-SD-NEXT: mov v0.16b, v2.16b
; CHECK-BF16SVE-SD-NEXT: ret
;
+; CHECK-CVT-GI-LABEL: test_fmul:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
; CHECK-BF16-GI-LABEL: test_fmul:
; CHECK-BF16-GI: // %bb.0:
-; CHECK-BF16-GI-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-GI-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-GI-NEXT: fmul v2.4s, v3.4s, v2.4s
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fmul v2.4s, v2.4s, v3.4s
; CHECK-BF16-GI-NEXT: fmul v1.4s, v0.4s, v1.4s
; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
@@ -315,39 +425,39 @@ define <8 x bfloat> @test_fmul(<8 x bfloat> %a, <8 x bfloat> %b) {
}
define <8 x bfloat> @test_fmadd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fmadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v6.4s, #127, msl #8
-; CHECK-CVT-NEXT: fmul v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: movi v4.4s, #1
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v5.16b, v4.16b
-; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: and v3.16b, v5.16b, v4.16b
-; CHECK-CVT-NEXT: addhn v1.4h, v1.4s, v6.4s
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: shll v3.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll2 v2.4s, v2.8h, #16
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v6.4s
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: fadd v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v2.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: and v1.16b, v3.16b, v4.16b
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v6.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v6.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v6.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: movi v4.4s, #1
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v5.16b, v4.16b
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: and v3.16b, v5.16b, v4.16b
+; CHECK-CVT-SD-NEXT: addhn v1.4h, v1.4s, v6.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: shll v3.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v6.4s
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v2.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: and v1.16b, v3.16b, v4.16b
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v6.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v6.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-SD-LABEL: test_fmadd:
; CHECK-BF16-SD: // %bb.0:
@@ -388,19 +498,75 @@ define <8 x bfloat> @test_fmadd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %
; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
; CHECK-BF16SVE-SD-NEXT: ret
;
+; CHECK-CVT-GI-LABEL: test_fmadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v7.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v16.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v17.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v6.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v18.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v19.4s, v0.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v6.16b, v6.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v16.16b, v16.16b
+; CHECK-CVT-GI-NEXT: add v5.4s, v17.4s, v5.4s
+; CHECK-CVT-GI-NEXT: mvn v17.16b, v18.16b
+; CHECK-CVT-GI-NEXT: add v6.4s, v19.4s, v6.4s
+; CHECK-CVT-GI-NEXT: bif v3.16b, v5.16b, v16.16b
+; CHECK-CVT-GI-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-CVT-GI-NEXT: bif v0.16b, v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: shrn v3.4h, v3.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd v3.4s, v3.4s, v5.4s
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v4.4s, v0.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v2.4s, v16.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v4.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bit v2.16b, v3.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
; CHECK-BF16-GI-LABEL: test_fmadd:
; CHECK-BF16-GI: // %bb.0:
-; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-BF16-GI-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v4.4s, v1.4h, #16
; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-GI-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fmul v3.4s, v3.4s, v4.4s
; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-BF16-GI-NEXT: bfcvtn v3.4h, v3.4s
-; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-GI-NEXT: shll v1.4s, v3.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvtn v1.4h, v3.4s
; CHECK-BF16-GI-NEXT: shll v3.4s, v2.4h, #16
; CHECK-BF16-GI-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
; CHECK-BF16-GI-NEXT: fadd v1.4s, v1.4s, v3.4s
; CHECK-BF16-GI-NEXT: fadd v2.4s, v0.4s, v2.4s
@@ -413,37 +579,93 @@ define <8 x bfloat> @test_fmadd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %
}
define <8 x bfloat> @test_fdiv(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: test_fdiv:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fdiv v2.4s, v3.4s, v2.4s
-; CHECK-CVT-NEXT: fdiv v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v0.16b
-; CHECK-CVT-NEXT: add v2.4s, v3.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v4.16b, v0.16b
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v5.4s
-; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fdiv:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-CVT-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-CVT-SD-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v4.16b, v0.16b
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v5.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fdiv:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fdiv v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fdiv v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fdiv:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fdiv:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fdiv:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fdiv v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fdiv:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fdiv v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
%r = fdiv <8 x bfloat> %a, %b
ret <8 x bfloat> %r
}
@@ -1863,140 +2085,222 @@ define <8 x bfloat> @test_bitcast_f16tobfloat(float, <8 x half> %a) {
}
define <8 x bfloat> @test_sqrt(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_sqrt:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: mov h1, v0.h[1]
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: mov h3, v0.h[2]
-; CHECK-CVT-NEXT: mov h4, v0.h[3]
-; CHECK-CVT-NEXT: mov h5, v0.h[4]
-; CHECK-CVT-NEXT: mov h6, v0.h[5]
-; CHECK-CVT-NEXT: mov h7, v0.h[6]
-; CHECK-CVT-NEXT: mov h0, v0.h[7]
-; CHECK-CVT-NEXT: fsqrt s2, s2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-CVT-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmov w10, s2
-; CHECK-CVT-NEXT: ubfx w12, w10, #16, #1
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: fsqrt s1, s1
-; CHECK-CVT-NEXT: fmov w9, s1
-; CHECK-CVT-NEXT: ubfx w11, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fsqrt s3, s3
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: fmov w11, s3
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w9, w11, w8
-; CHECK-CVT-NEXT: add w9, w12, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fsqrt s4, s4
-; CHECK-CVT-NEXT: fsqrt s5, s5
-; CHECK-CVT-NEXT: fsqrt s6, s6
-; CHECK-CVT-NEXT: fsqrt s7, s7
-; CHECK-CVT-NEXT: fsqrt s1, s0
-; CHECK-CVT-NEXT: fmov s0, w10
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: mov v0.h[1], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s5
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[2], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s6
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[3], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s7
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[4], v2.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v0.h[5], v2.h[0]
-; CHECK-CVT-NEXT: fmov w10, s1
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: mov v0.h[6], v1.h[0]
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s1, w8
-; CHECK-CVT-NEXT: mov v0.h[7], v1.h[0]
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sqrt:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: mov h1, v0.h[1]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h4, v0.h[3]
+; CHECK-CVT-SD-NEXT: mov h5, v0.h[4]
+; CHECK-CVT-SD-NEXT: mov h6, v0.h[5]
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[6]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[7]
+; CHECK-CVT-SD-NEXT: fsqrt s2, s2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w10, s2
+; CHECK-CVT-SD-NEXT: ubfx w12, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: fsqrt s1, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s1
+; CHECK-CVT-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fsqrt s3, s3
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: fmov w11, s3
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w11, w8
+; CHECK-CVT-SD-NEXT: add w9, w12, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fsqrt s4, s4
+; CHECK-CVT-SD-NEXT: fsqrt s5, s5
+; CHECK-CVT-SD-NEXT: fsqrt s6, s6
+; CHECK-CVT-SD-NEXT: fsqrt s7, s7
+; CHECK-CVT-SD-NEXT: fsqrt s1, s0
+; CHECK-CVT-SD-NEXT: fmov s0, w10
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: mov v0.h[1], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s5
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[2], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s6
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[3], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s7
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[4], v2.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v0.h[5], v2.h[0]
+; CHECK-CVT-SD-NEXT: fmov w10, s1
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[6], v1.h[0]
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s1, w8
+; CHECK-CVT-SD-NEXT: mov v0.h[7], v1.h[0]
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_sqrt:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: mov h1, v0.h[1]
-; CHECK-BF16-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h3, v0.h[2]
-; CHECK-BF16-NEXT: mov h4, v0.h[3]
-; CHECK-BF16-NEXT: mov h5, v0.h[4]
-; CHECK-BF16-NEXT: mov h6, v0.h[5]
-; CHECK-BF16-NEXT: mov h7, v0.h[6]
-; CHECK-BF16-NEXT: mov h0, v0.h[7]
-; CHECK-BF16-NEXT: fsqrt s2, s2
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v0.4h, #16
-; CHECK-BF16-NEXT: bfcvt h0, s2
-; CHECK-BF16-NEXT: fsqrt s1, s1
-; CHECK-BF16-NEXT: bfcvt h1, s1
-; CHECK-BF16-NEXT: mov v0.h[1], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s3, s3
-; CHECK-BF16-NEXT: bfcvt h1, s3
-; CHECK-BF16-NEXT: mov v0.h[2], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s4, s4
-; CHECK-BF16-NEXT: bfcvt h1, s4
-; CHECK-BF16-NEXT: mov v0.h[3], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s5, s5
-; CHECK-BF16-NEXT: bfcvt h1, s5
-; CHECK-BF16-NEXT: mov v0.h[4], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s6, s6
-; CHECK-BF16-NEXT: bfcvt h1, s6
-; CHECK-BF16-NEXT: mov v0.h[5], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s7, s7
-; CHECK-BF16-NEXT: bfcvt h1, s7
-; CHECK-BF16-NEXT: mov v0.h[6], v1.h[0]
-; CHECK-BF16-NEXT: fsqrt s2, s16
-; CHECK-BF16-NEXT: bfcvt h1, s2
-; CHECK-BF16-NEXT: mov v0.h[7], v1.h[0]
-; CHECK-BF16-NEXT: ret
- %r = call <8 x bfloat> @llvm.sqrt.v8bf16(<8 x bfloat> %a)
- ret <8 x bfloat> %r
-}
-
-define <8 x bfloat> @test_powi(<8 x bfloat> %a, i32 %b) #0 {
-; CHECK-CVT-LABEL: test_powi:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: sub sp, sp, #64
-; CHECK-CVT-NEXT: mov h1, v0.h[1]
-; CHECK-CVT-NEXT: str q0, [sp] // 16-byte Spill
+; CHECK-BF16-SD-LABEL: test_sqrt:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: mov h1, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h4, v0.h[3]
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[4]
+; CHECK-BF16-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[6]
+; CHECK-BF16-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16-SD-NEXT: fsqrt s2, s2
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h0, s2
+; CHECK-BF16-SD-NEXT: fsqrt s1, s1
+; CHECK-BF16-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s3, s3
+; CHECK-BF16-SD-NEXT: bfcvt h1, s3
+; CHECK-BF16-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s4, s4
+; CHECK-BF16-SD-NEXT: bfcvt h1, s4
+; CHECK-BF16-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s5, s5
+; CHECK-BF16-SD-NEXT: bfcvt h1, s5
+; CHECK-BF16-SD-NEXT: mov v0.h[4], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s6, s6
+; CHECK-BF16-SD-NEXT: bfcvt h1, s6
+; CHECK-BF16-SD-NEXT: mov v0.h[5], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s7, s7
+; CHECK-BF16-SD-NEXT: bfcvt h1, s7
+; CHECK-BF16-SD-NEXT: mov v0.h[6], v1.h[0]
+; CHECK-BF16-SD-NEXT: fsqrt s2, s16
+; CHECK-BF16-SD-NEXT: bfcvt h1, s2
+; CHECK-BF16-SD-NEXT: mov v0.h[7], v1.h[0]
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_sqrt:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: mov h1, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h3, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h4, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT: mov h5, v0.h[4]
+; CHECK-BF16SVE-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16SVE-SD-NEXT: mov h7, v0.h[6]
+; CHECK-BF16SVE-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s2, s2
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: bfcvt h0, s2
+; CHECK-BF16SVE-SD-NEXT: fsqrt s1, s1
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s1
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[1], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s3, s3
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s3
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[2], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s4, s4
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s4
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[3], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s5, s5
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s5
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[4], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s6, s6
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s6
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[5], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s7, s7
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s7
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[6], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: fsqrt s2, s16
+; CHECK-BF16SVE-SD-NEXT: bfcvt h1, s2
+; CHECK-BF16SVE-SD-NEXT: mov v0.h[7], v1.h[0]
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sqrt:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsqrt v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fsqrt v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_sqrt:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: fsqrt v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: fsqrt v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
+ %r = call <8 x bfloat> @llvm.sqrt.v8bf16(<8 x bfloat> %a)
+ ret <8 x bfloat> %r
+}
+
+define <8 x bfloat> @test_powi(<8 x bfloat> %a, i32 %b) #0 {
+; CHECK-CVT-LABEL: test_powi:
+; CHECK-CVT: // %bb.0:
+; CHECK-CVT-NEXT: sub sp, sp, #64
+; CHECK-CVT-NEXT: mov h1, v0.h[1]
+; CHECK-CVT-NEXT: str q0, [sp] // 16-byte Spill
; CHECK-CVT-NEXT: str x30, [sp, #32] // 8-byte Spill
; CHECK-CVT-NEXT: stp x20, x19, [sp, #48] // 16-byte Folded Spill
; CHECK-CVT-NEXT: mov w19, w0
@@ -5451,192 +5755,314 @@ define <8 x bfloat> @test_log2(<8 x bfloat> %a) #0 {
}
define <8 x bfloat> @test_fma(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fma:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: mov h3, v2.h[1]
-; CHECK-CVT-NEXT: mov h4, v1.h[1]
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: mov h5, v0.h[1]
-; CHECK-CVT-NEXT: shll v6.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v7.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v16.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h17, v2.h[2]
-; CHECK-CVT-NEXT: mov h18, v1.h[2]
-; CHECK-CVT-NEXT: mov h19, v0.h[2]
-; CHECK-CVT-NEXT: mov h20, v2.h[3]
-; CHECK-CVT-NEXT: mov h21, v1.h[3]
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-CVT-NEXT: mov h22, v0.h[3]
-; CHECK-CVT-NEXT: fmadd s6, s16, s7, s6
-; CHECK-CVT-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-CVT-NEXT: shll v7.4s, v17.4h, #16
-; CHECK-CVT-NEXT: shll v16.4s, v18.4h, #16
-; CHECK-CVT-NEXT: shll v17.4s, v19.4h, #16
-; CHECK-CVT-NEXT: shll v18.4s, v20.4h, #16
-; CHECK-CVT-NEXT: shll v19.4s, v21.4h, #16
-; CHECK-CVT-NEXT: fmadd s3, s5, s4, s3
-; CHECK-CVT-NEXT: shll v20.4s, v22.4h, #16
-; CHECK-CVT-NEXT: mov h21, v2.h[4]
-; CHECK-CVT-NEXT: fmov w9, s6
-; CHECK-CVT-NEXT: mov h22, v1.h[4]
-; CHECK-CVT-NEXT: mov h6, v0.h[4]
-; CHECK-CVT-NEXT: fmadd s4, s17, s16, s7
-; CHECK-CVT-NEXT: mov h17, v2.h[5]
-; CHECK-CVT-NEXT: fmadd s5, s20, s19, s18
-; CHECK-CVT-NEXT: mov h18, v1.h[5]
-; CHECK-CVT-NEXT: mov h19, v0.h[5]
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: fmov w11, s3
-; CHECK-CVT-NEXT: shll v7.4s, v21.4h, #16
-; CHECK-CVT-NEXT: shll v16.4s, v22.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v6.4h, #16
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: shll v6.4s, v17.4h, #16
-; CHECK-CVT-NEXT: mov h17, v1.h[6]
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w10, w11, w8
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: fmov w11, s4
-; CHECK-CVT-NEXT: fmadd s4, s3, s16, s7
-; CHECK-CVT-NEXT: shll v7.4s, v18.4h, #16
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: fmov s3, w9
-; CHECK-CVT-NEXT: fmov w9, s5
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: shll v16.4s, v19.4h, #16
-; CHECK-CVT-NEXT: mov h18, v0.h[6]
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w11, w11, w8
-; CHECK-CVT-NEXT: mov h1, v1.h[7]
-; CHECK-CVT-NEXT: fmov s5, w10
-; CHECK-CVT-NEXT: mov h0, v0.h[7]
-; CHECK-CVT-NEXT: add w11, w12, w11
-; CHECK-CVT-NEXT: fmov w12, s4
-; CHECK-CVT-NEXT: fmadd s6, s16, s7, s6
-; CHECK-CVT-NEXT: lsr w10, w11, #16
-; CHECK-CVT-NEXT: ubfx w11, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: mov v3.h[1], v5.h[0]
-; CHECK-CVT-NEXT: mov h5, v2.h[6]
-; CHECK-CVT-NEXT: shll v7.4s, v18.4h, #16
-; CHECK-CVT-NEXT: fmov s4, w10
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: mov h2, v2.h[7]
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: ubfx w10, w12, #16, #1
-; CHECK-CVT-NEXT: fmov w11, s6
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov v3.h[2], v4.h[0]
-; CHECK-CVT-NEXT: shll v4.4s, v5.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v17.4h, #16
-; CHECK-CVT-NEXT: fmov s16, w9
-; CHECK-CVT-NEXT: add w9, w12, w8
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: ubfx w10, w11, #16, #1
-; CHECK-CVT-NEXT: fmadd s4, s7, s5, s4
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v3.h[3], v16.h[0]
-; CHECK-CVT-NEXT: fmadd s0, s0, s1, s2
-; CHECK-CVT-NEXT: fmov s5, w9
-; CHECK-CVT-NEXT: add w9, w11, w8
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v3.h[4], v5.h[0]
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s0
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: mov v3.h[5], v1.h[0]
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: fmov s0, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v3.h[6], v0.h[0]
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: mov v3.h[7], v0.h[0]
-; CHECK-CVT-NEXT: mov v0.16b, v3.16b
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fma:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: mov h3, v2.h[1]
+; CHECK-CVT-SD-NEXT: mov h4, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: mov h5, v0.h[1]
+; CHECK-CVT-SD-NEXT: shll v6.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h17, v2.h[2]
+; CHECK-CVT-SD-NEXT: mov h18, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h19, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h20, v2.h[3]
+; CHECK-CVT-SD-NEXT: mov h21, v1.h[3]
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT: mov h22, v0.h[3]
+; CHECK-CVT-SD-NEXT: fmadd s6, s16, s7, s6
+; CHECK-CVT-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: shll v7.4s, v17.4h, #16
+; CHECK-CVT-SD-NEXT: shll v16.4s, v18.4h, #16
+; CHECK-CVT-SD-NEXT: shll v17.4s, v19.4h, #16
+; CHECK-CVT-SD-NEXT: shll v18.4s, v20.4h, #16
+; CHECK-CVT-SD-NEXT: shll v19.4s, v21.4h, #16
+; CHECK-CVT-SD-NEXT: fmadd s3, s5, s4, s3
+; CHECK-CVT-SD-NEXT: shll v20.4s, v22.4h, #16
+; CHECK-CVT-SD-NEXT: mov h21, v2.h[4]
+; CHECK-CVT-SD-NEXT: fmov w9, s6
+; CHECK-CVT-SD-NEXT: mov h22, v1.h[4]
+; CHECK-CVT-SD-NEXT: mov h6, v0.h[4]
+; CHECK-CVT-SD-NEXT: fmadd s4, s17, s16, s7
+; CHECK-CVT-SD-NEXT: mov h17, v2.h[5]
+; CHECK-CVT-SD-NEXT: fmadd s5, s20, s19, s18
+; CHECK-CVT-SD-NEXT: mov h18, v1.h[5]
+; CHECK-CVT-SD-NEXT: mov h19, v0.h[5]
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: fmov w11, s3
+; CHECK-CVT-SD-NEXT: shll v7.4s, v21.4h, #16
+; CHECK-CVT-SD-NEXT: shll v16.4s, v22.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: shll v6.4s, v17.4h, #16
+; CHECK-CVT-SD-NEXT: mov h17, v1.h[6]
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w10, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: fmov w11, s4
+; CHECK-CVT-SD-NEXT: fmadd s4, s3, s16, s7
+; CHECK-CVT-SD-NEXT: shll v7.4s, v18.4h, #16
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: fmov s3, w9
+; CHECK-CVT-SD-NEXT: fmov w9, s5
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: shll v16.4s, v19.4h, #16
+; CHECK-CVT-SD-NEXT: mov h18, v0.h[6]
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w11, w11, w8
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[7]
+; CHECK-CVT-SD-NEXT: fmov s5, w10
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[7]
+; CHECK-CVT-SD-NEXT: add w11, w12, w11
+; CHECK-CVT-SD-NEXT: fmov w12, s4
+; CHECK-CVT-SD-NEXT: fmadd s6, s16, s7, s6
+; CHECK-CVT-SD-NEXT: lsr w10, w11, #16
+; CHECK-CVT-SD-NEXT: ubfx w11, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: mov v3.h[1], v5.h[0]
+; CHECK-CVT-SD-NEXT: mov h5, v2.h[6]
+; CHECK-CVT-SD-NEXT: shll v7.4s, v18.4h, #16
+; CHECK-CVT-SD-NEXT: fmov s4, w10
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: mov h2, v2.h[7]
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: ubfx w10, w12, #16, #1
+; CHECK-CVT-SD-NEXT: fmov w11, s6
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov v3.h[2], v4.h[0]
+; CHECK-CVT-SD-NEXT: shll v4.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v17.4h, #16
+; CHECK-CVT-SD-NEXT: fmov s16, w9
+; CHECK-CVT-SD-NEXT: add w9, w12, w8
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: ubfx w10, w11, #16, #1
+; CHECK-CVT-SD-NEXT: fmadd s4, s7, s5, s4
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v3.h[3], v16.h[0]
+; CHECK-CVT-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-CVT-SD-NEXT: fmov s5, w9
+; CHECK-CVT-SD-NEXT: add w9, w11, w8
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v3.h[4], v5.h[0]
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s0
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: mov v3.h[5], v1.h[0]
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s0, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v3.h[6], v0.h[0]
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: mov v3.h[7], v0.h[0]
+; CHECK-CVT-SD-NEXT: mov v0.16b, v3.16b
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fma:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: mov h3, v2.h[1]
-; CHECK-BF16-NEXT: mov h4, v1.h[1]
-; CHECK-BF16-NEXT: mov h5, v0.h[1]
-; CHECK-BF16-NEXT: shll v6.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v7.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h17, v2.h[2]
-; CHECK-BF16-NEXT: mov h18, v1.h[2]
-; CHECK-BF16-NEXT: mov h19, v0.h[2]
-; CHECK-BF16-NEXT: mov h20, v2.h[3]
-; CHECK-BF16-NEXT: mov h21, v1.h[3]
-; CHECK-BF16-NEXT: fmadd s6, s16, s7, s6
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: mov h7, v0.h[3]
-; CHECK-BF16-NEXT: shll v16.4s, v19.4h, #16
-; CHECK-BF16-NEXT: mov h19, v0.h[4]
-; CHECK-BF16-NEXT: fmadd s4, s5, s4, s3
-; CHECK-BF16-NEXT: shll v3.4s, v17.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v18.4h, #16
-; CHECK-BF16-NEXT: mov h17, v2.h[4]
-; CHECK-BF16-NEXT: mov h18, v1.h[4]
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: shll v19.4s, v19.4h, #16
-; CHECK-BF16-NEXT: fmadd s5, s16, s5, s3
-; CHECK-BF16-NEXT: bfcvt h3, s6
-; CHECK-BF16-NEXT: shll v6.4s, v20.4h, #16
-; CHECK-BF16-NEXT: bfcvt h4, s4
-; CHECK-BF16-NEXT: shll v16.4s, v21.4h, #16
-; CHECK-BF16-NEXT: shll v17.4s, v17.4h, #16
-; CHECK-BF16-NEXT: shll v18.4s, v18.4h, #16
-; CHECK-BF16-NEXT: fmadd s6, s7, s16, s6
-; CHECK-BF16-NEXT: bfcvt h5, s5
-; CHECK-BF16-NEXT: mov h7, v1.h[5]
-; CHECK-BF16-NEXT: mov v3.h[1], v4.h[0]
-; CHECK-BF16-NEXT: mov h4, v2.h[5]
-; CHECK-BF16-NEXT: mov h16, v0.h[5]
-; CHECK-BF16-NEXT: fmadd s17, s19, s18, s17
-; CHECK-BF16-NEXT: mov h18, v2.h[6]
-; CHECK-BF16-NEXT: mov h19, v1.h[6]
-; CHECK-BF16-NEXT: mov h2, v2.h[7]
-; CHECK-BF16-NEXT: mov h1, v1.h[7]
-; CHECK-BF16-NEXT: bfcvt h6, s6
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: mov v3.h[2], v5.h[0]
-; CHECK-BF16-NEXT: mov h5, v0.h[6]
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v16.4h, #16
-; CHECK-BF16-NEXT: mov h0, v0.h[7]
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fmadd s4, s16, s7, s4
-; CHECK-BF16-NEXT: mov v3.h[3], v6.h[0]
-; CHECK-BF16-NEXT: bfcvt h6, s17
-; CHECK-BF16-NEXT: shll v7.4s, v18.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v19.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmadd s5, s5, s16, s7
-; CHECK-BF16-NEXT: mov v3.h[4], v6.h[0]
-; CHECK-BF16-NEXT: bfcvt h4, s4
-; CHECK-BF16-NEXT: fmadd s0, s0, s1, s2
-; CHECK-BF16-NEXT: mov v3.h[5], v4.h[0]
-; CHECK-BF16-NEXT: bfcvt h4, s5
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: mov v3.h[6], v4.h[0]
-; CHECK-BF16-NEXT: mov v3.h[7], v0.h[0]
-; CHECK-BF16-NEXT: mov v0.16b, v3.16b
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fma:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: mov h3, v2.h[1]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h17, v2.h[2]
+; CHECK-BF16-SD-NEXT: mov h18, v1.h[2]
+; CHECK-BF16-SD-NEXT: mov h19, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h20, v2.h[3]
+; CHECK-BF16-SD-NEXT: mov h21, v1.h[3]
+; CHECK-BF16-SD-NEXT: fmadd s6, s16, s7, s6
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[3]
+; CHECK-BF16-SD-NEXT: shll v16.4s, v19.4h, #16
+; CHECK-BF16-SD-NEXT: mov h19, v0.h[4]
+; CHECK-BF16-SD-NEXT: fmadd s4, s5, s4, s3
+; CHECK-BF16-SD-NEXT: shll v3.4s, v17.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v18.4h, #16
+; CHECK-BF16-SD-NEXT: mov h17, v2.h[4]
+; CHECK-BF16-SD-NEXT: mov h18, v1.h[4]
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: shll v19.4s, v19.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s5, s16, s5, s3
+; CHECK-BF16-SD-NEXT: bfcvt h3, s6
+; CHECK-BF16-SD-NEXT: shll v6.4s, v20.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16-SD-NEXT: shll v16.4s, v21.4h, #16
+; CHECK-BF16-SD-NEXT: shll v17.4s, v17.4h, #16
+; CHECK-BF16-SD-NEXT: shll v18.4s, v18.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s6, s7, s16, s6
+; CHECK-BF16-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16-SD-NEXT: mov h7, v1.h[5]
+; CHECK-BF16-SD-NEXT: mov v3.h[1], v4.h[0]
+; CHECK-BF16-SD-NEXT: mov h4, v2.h[5]
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[5]
+; CHECK-BF16-SD-NEXT: fmadd s17, s19, s18, s17
+; CHECK-BF16-SD-NEXT: mov h18, v2.h[6]
+; CHECK-BF16-SD-NEXT: mov h19, v1.h[6]
+; CHECK-BF16-SD-NEXT: mov h2, v2.h[7]
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16-SD-NEXT: bfcvt h6, s6
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: mov v3.h[2], v5.h[0]
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[6]
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s4, s16, s7, s4
+; CHECK-BF16-SD-NEXT: mov v3.h[3], v6.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h6, s17
+; CHECK-BF16-SD-NEXT: shll v7.4s, v18.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v19.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s5, s5, s16, s7
+; CHECK-BF16-SD-NEXT: mov v3.h[4], v6.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-SD-NEXT: mov v3.h[5], v4.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h4, s5
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: mov v3.h[6], v4.h[0]
+; CHECK-BF16-SD-NEXT: mov v3.h[7], v0.h[0]
+; CHECK-BF16-SD-NEXT: mov v0.16b, v3.16b
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fma:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: mov h3, v2.h[1]
+; CHECK-BF16SVE-SD-NEXT: mov h4, v1.h[1]
+; CHECK-BF16SVE-SD-NEXT: mov h5, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h17, v2.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h18, v1.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h19, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h20, v2.h[3]
+; CHECK-BF16SVE-SD-NEXT: mov h21, v1.h[3]
+; CHECK-BF16SVE-SD-NEXT: fmadd s6, s16, s7, s6
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h7, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v19.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h19, v0.h[4]
+; CHECK-BF16SVE-SD-NEXT: fmadd s4, s5, s4, s3
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v17.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v18.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h17, v2.h[4]
+; CHECK-BF16SVE-SD-NEXT: mov h18, v1.h[4]
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v19.4s, v19.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmadd s5, s16, s5, s3
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s6
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v20.4h, #16
+; CHECK-BF16SVE-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v21.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v17.4s, v17.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v18.4s, v18.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmadd s6, s7, s16, s6
+; CHECK-BF16SVE-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16SVE-SD-NEXT: mov h7, v1.h[5]
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[1], v4.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h4, v2.h[5]
+; CHECK-BF16SVE-SD-NEXT: mov h16, v0.h[5]
+; CHECK-BF16SVE-SD-NEXT: fmadd s17, s19, s18, s17
+; CHECK-BF16SVE-SD-NEXT: mov h18, v2.h[6]
+; CHECK-BF16SVE-SD-NEXT: mov h19, v1.h[6]
+; CHECK-BF16SVE-SD-NEXT: mov h2, v2.h[7]
+; CHECK-BF16SVE-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h6, s6
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[2], v5.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h5, v0.h[6]
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmadd s4, s16, s7, s4
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[3], v6.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h6, s17
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v18.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v19.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmadd s5, s5, s16, s7
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[4], v6.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16SVE-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[5], v4.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h4, s5
+; CHECK-BF16SVE-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[6], v4.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v3.h[7], v0.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v0.16b, v3.16b
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fma:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmla v5.4s, v4.4s, v3.4s
+; CHECK-CVT-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: movi v0.4s, #1
+; CHECK-CVT-GI-NEXT: movi v1.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v5.4s, v5.4s
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v5.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v5.4s, v1.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-CVT-GI-NEXT: and v0.16b, v4.16b, v0.16b
+; CHECK-CVT-GI-NEXT: orr v4.16b, v5.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v3.4s, v16.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT: mov v1.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bsl v1.16b, v4.16b, v3.16b
+; CHECK-CVT-GI-NEXT: bif v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fma:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-BF16-GI-NEXT: fmla v5.4s, v4.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fmla v2.4s, v1.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v5.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.fma.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c)
ret <8 x bfloat> %r
}
@@ -5661,323 +6087,527 @@ define <8 x bfloat> @test_fneg(<8 x bfloat> %a) #0 {
}
define <8 x bfloat> @test_minnum(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
-; CHECK-CVT-LABEL: test_minnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: mov h2, v1.h[1]
-; CHECK-CVT-NEXT: mov h3, v0.h[1]
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h6, v1.h[2]
-; CHECK-CVT-NEXT: mov h7, v0.h[2]
-; CHECK-CVT-NEXT: mov h16, v1.h[3]
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: fminnm s4, s5, s4
-; CHECK-CVT-NEXT: shll v5.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v6.4s, v7.4h, #16
-; CHECK-CVT-NEXT: mov h7, v1.h[4]
-; CHECK-CVT-NEXT: shll v16.4s, v16.4h, #16
-; CHECK-CVT-NEXT: fminnm s2, s3, s2
-; CHECK-CVT-NEXT: mov h3, v0.h[3]
-; CHECK-CVT-NEXT: fmov w9, s4
-; CHECK-CVT-NEXT: mov h4, v0.h[4]
-; CHECK-CVT-NEXT: fminnm s5, s6, s5
-; CHECK-CVT-NEXT: shll v6.4s, v7.4h, #16
-; CHECK-CVT-NEXT: fmov w11, s2
-; CHECK-CVT-NEXT: shll v2.4s, v3.4h, #16
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: mov h3, v1.h[5]
-; CHECK-CVT-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w10, w11, w8
-; CHECK-CVT-NEXT: fminnm s7, s2, s16
-; CHECK-CVT-NEXT: fmov w11, s5
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov h5, v0.h[5]
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: fminnm s4, s4, s6
-; CHECK-CVT-NEXT: mov h6, v1.h[6]
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: ubfx w9, w11, #16, #1
-; CHECK-CVT-NEXT: add w11, w11, w8
-; CHECK-CVT-NEXT: mov h1, v1.h[7]
-; CHECK-CVT-NEXT: fmov s16, w10
-; CHECK-CVT-NEXT: fmov w10, s7
-; CHECK-CVT-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-CVT-NEXT: add w9, w9, w11
-; CHECK-CVT-NEXT: mov h7, v0.h[6]
-; CHECK-CVT-NEXT: mov h0, v0.h[7]
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: mov v2.h[1], v16.h[0]
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: fmov s16, w9
-; CHECK-CVT-NEXT: fminnm s3, s5, s3
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: add w9, w11, w10
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: shll v4.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov v2.h[2], v16.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s6, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: fminnm s3, s5, s4
-; CHECK-CVT-NEXT: fminnm s0, s0, s1
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[3], v6.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s4, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[4], v4.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s0
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[5], v1.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: fmov s0, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v2.h[6], v0.h[0]
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT: mov v0.16b, v2.16b
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_minnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: mov h2, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h16, v1.h[3]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: fminnm s4, s5, s4
+; CHECK-CVT-SD-NEXT: shll v5.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v6.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: mov h7, v1.h[4]
+; CHECK-CVT-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-CVT-SD-NEXT: fminnm s2, s3, s2
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[3]
+; CHECK-CVT-SD-NEXT: fmov w9, s4
+; CHECK-CVT-SD-NEXT: mov h4, v0.h[4]
+; CHECK-CVT-SD-NEXT: fminnm s5, s6, s5
+; CHECK-CVT-SD-NEXT: shll v6.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w11, s2
+; CHECK-CVT-SD-NEXT: shll v2.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: mov h3, v1.h[5]
+; CHECK-CVT-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w10, w11, w8
+; CHECK-CVT-SD-NEXT: fminnm s7, s2, s16
+; CHECK-CVT-SD-NEXT: fmov w11, s5
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov h5, v0.h[5]
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: fminnm s4, s4, s6
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[6]
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: ubfx w9, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w11, w11, w8
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[7]
+; CHECK-CVT-SD-NEXT: fmov s16, w10
+; CHECK-CVT-SD-NEXT: fmov w10, s7
+; CHECK-CVT-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w9, w11
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[6]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[7]
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: mov v2.h[1], v16.h[0]
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s16, w9
+; CHECK-CVT-SD-NEXT: fminnm s3, s5, s3
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w11, w10
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: shll v4.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[2], v16.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s6, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: fminnm s3, s5, s4
+; CHECK-CVT-SD-NEXT: fminnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[3], v6.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s4, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[4], v4.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s0
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[5], v1.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s0, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[6], v0.h[0]
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-CVT-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_minnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: mov h2, v1.h[1]
-; CHECK-BF16-NEXT: mov h3, v0.h[1]
-; CHECK-BF16-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h6, v1.h[2]
-; CHECK-BF16-NEXT: mov h7, v0.h[2]
-; CHECK-BF16-NEXT: mov h16, v1.h[3]
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: fminnm s4, s5, s4
-; CHECK-BF16-NEXT: mov h5, v0.h[3]
-; CHECK-BF16-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: fminnm s3, s3, s2
-; CHECK-BF16-NEXT: bfcvt h2, s4
-; CHECK-BF16-NEXT: fminnm s4, s7, s6
-; CHECK-BF16-NEXT: shll v6.4s, v16.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: mov h7, v1.h[4]
-; CHECK-BF16-NEXT: mov h16, v0.h[4]
-; CHECK-BF16-NEXT: bfcvt h3, s3
-; CHECK-BF16-NEXT: fminnm s5, s5, s6
-; CHECK-BF16-NEXT: bfcvt h4, s4
-; CHECK-BF16-NEXT: mov h6, v0.h[5]
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v16.4h, #16
-; CHECK-BF16-NEXT: mov v2.h[1], v3.h[0]
-; CHECK-BF16-NEXT: mov h3, v1.h[5]
-; CHECK-BF16-NEXT: bfcvt h5, s5
-; CHECK-BF16-NEXT: fminnm s7, s16, s7
-; CHECK-BF16-NEXT: mov h16, v0.h[6]
-; CHECK-BF16-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-BF16-NEXT: mov h0, v0.h[7]
-; CHECK-BF16-NEXT: mov v2.h[2], v4.h[0]
-; CHECK-BF16-NEXT: mov h4, v1.h[6]
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: mov h1, v1.h[7]
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fminnm s3, s6, s3
-; CHECK-BF16-NEXT: shll v6.4s, v16.4h, #16
-; CHECK-BF16-NEXT: mov v2.h[3], v5.h[0]
-; CHECK-BF16-NEXT: bfcvt h5, s7
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fminnm s4, s6, s4
-; CHECK-BF16-NEXT: bfcvt h3, s3
-; CHECK-BF16-NEXT: mov v2.h[4], v5.h[0]
-; CHECK-BF16-NEXT: fminnm s0, s0, s1
-; CHECK-BF16-NEXT: mov v2.h[5], v3.h[0]
-; CHECK-BF16-NEXT: bfcvt h3, s4
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: mov v2.h[6], v3.h[0]
-; CHECK-BF16-NEXT: mov v2.h[7], v0.h[0]
-; CHECK-BF16-NEXT: mov v0.16b, v2.16b
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_minnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h6, v1.h[2]
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h16, v1.h[3]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: fminnm s4, s5, s4
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[3]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: fminnm s3, s3, s2
+; CHECK-BF16-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16-SD-NEXT: fminnm s4, s7, s6
+; CHECK-BF16-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: mov h7, v1.h[4]
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[4]
+; CHECK-BF16-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16-SD-NEXT: fminnm s5, s5, s6
+; CHECK-BF16-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: mov v2.h[1], v3.h[0]
+; CHECK-BF16-SD-NEXT: mov h3, v1.h[5]
+; CHECK-BF16-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16-SD-NEXT: fminnm s7, s16, s7
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[6]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16-SD-NEXT: mov v2.h[2], v4.h[0]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[6]
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fminnm s3, s6, s3
+; CHECK-BF16-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: mov v2.h[3], v5.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h5, s7
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fminnm s4, s6, s4
+; CHECK-BF16-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16-SD-NEXT: mov v2.h[4], v5.h[0]
+; CHECK-BF16-SD-NEXT: fminnm s0, s0, s1
+; CHECK-BF16-SD-NEXT: mov v2.h[5], v3.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h3, s4
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: mov v2.h[6], v3.h[0]
+; CHECK-BF16-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-BF16-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_minnum:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16SVE-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h6, v1.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h16, v1.h[3]
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fminnm s4, s5, s4
+; CHECK-BF16SVE-SD-NEXT: mov h5, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fminnm s3, s3, s2
+; CHECK-BF16SVE-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16SVE-SD-NEXT: fminnm s4, s7, s6
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h7, v1.h[4]
+; CHECK-BF16SVE-SD-NEXT: mov h16, v0.h[4]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16SVE-SD-NEXT: fminnm s5, s5, s6
+; CHECK-BF16SVE-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16SVE-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[1], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h3, v1.h[5]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16SVE-SD-NEXT: fminnm s7, s16, s7
+; CHECK-BF16SVE-SD-NEXT: mov h16, v0.h[6]
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[2], v4.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h4, v1.h[6]
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16SVE-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fminnm s3, s6, s3
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[3], v5.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h5, s7
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fminnm s4, s6, s4
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[4], v5.h[0]
+; CHECK-BF16SVE-SD-NEXT: fminnm s0, s0, s1
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[5], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s4
+; CHECK-BF16SVE-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[6], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_minnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fminnm v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fminnm v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_minnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fminnm v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fminnm v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.minnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_maxnum(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
-; CHECK-CVT-LABEL: test_maxnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: mov h2, v1.h[1]
-; CHECK-CVT-NEXT: mov h3, v0.h[1]
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov h6, v1.h[2]
-; CHECK-CVT-NEXT: mov h7, v0.h[2]
-; CHECK-CVT-NEXT: mov h16, v1.h[3]
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s4, s5, s4
-; CHECK-CVT-NEXT: shll v5.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v6.4s, v7.4h, #16
-; CHECK-CVT-NEXT: mov h7, v1.h[4]
-; CHECK-CVT-NEXT: shll v16.4s, v16.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s2, s3, s2
-; CHECK-CVT-NEXT: mov h3, v0.h[3]
-; CHECK-CVT-NEXT: fmov w9, s4
-; CHECK-CVT-NEXT: mov h4, v0.h[4]
-; CHECK-CVT-NEXT: fmaxnm s5, s6, s5
-; CHECK-CVT-NEXT: shll v6.4s, v7.4h, #16
-; CHECK-CVT-NEXT: fmov w11, s2
-; CHECK-CVT-NEXT: shll v2.4s, v3.4h, #16
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w9, w9, w8
-; CHECK-CVT-NEXT: mov h3, v1.h[5]
-; CHECK-CVT-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-CVT-NEXT: add w9, w10, w9
-; CHECK-CVT-NEXT: ubfx w12, w11, #16, #1
-; CHECK-CVT-NEXT: add w10, w11, w8
-; CHECK-CVT-NEXT: fmaxnm s7, s2, s16
-; CHECK-CVT-NEXT: fmov w11, s5
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov h5, v0.h[5]
-; CHECK-CVT-NEXT: add w10, w12, w10
-; CHECK-CVT-NEXT: fmaxnm s4, s4, s6
-; CHECK-CVT-NEXT: mov h6, v1.h[6]
-; CHECK-CVT-NEXT: lsr w10, w10, #16
-; CHECK-CVT-NEXT: fmov s2, w9
-; CHECK-CVT-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-CVT-NEXT: ubfx w9, w11, #16, #1
-; CHECK-CVT-NEXT: add w11, w11, w8
-; CHECK-CVT-NEXT: mov h1, v1.h[7]
-; CHECK-CVT-NEXT: fmov s16, w10
-; CHECK-CVT-NEXT: fmov w10, s7
-; CHECK-CVT-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-CVT-NEXT: add w9, w9, w11
-; CHECK-CVT-NEXT: mov h7, v0.h[6]
-; CHECK-CVT-NEXT: mov h0, v0.h[7]
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: mov v2.h[1], v16.h[0]
-; CHECK-CVT-NEXT: add w10, w10, w8
-; CHECK-CVT-NEXT: fmov s16, w9
-; CHECK-CVT-NEXT: fmaxnm s3, s5, s3
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: add w9, w11, w10
-; CHECK-CVT-NEXT: fmov w10, s4
-; CHECK-CVT-NEXT: shll v4.4s, v6.4h, #16
-; CHECK-CVT-NEXT: shll v5.4s, v7.4h, #16
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: mov v2.h[2], v16.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s6, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: fmaxnm s3, s5, s4
-; CHECK-CVT-NEXT: fmaxnm s0, s0, s1
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[3], v6.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s4, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s3
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[4], v4.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: fmov s1, w9
-; CHECK-CVT-NEXT: add w9, w10, w8
-; CHECK-CVT-NEXT: fmov w10, s0
-; CHECK-CVT-NEXT: add w9, w11, w9
-; CHECK-CVT-NEXT: lsr w9, w9, #16
-; CHECK-CVT-NEXT: mov v2.h[5], v1.h[0]
-; CHECK-CVT-NEXT: ubfx w11, w10, #16, #1
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: fmov s0, w9
-; CHECK-CVT-NEXT: add w8, w11, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: mov v2.h[6], v0.h[0]
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: mov v2.h[7], v0.h[0]
-; CHECK-CVT-NEXT: mov v0.16b, v2.16b
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_maxnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: mov h2, v1.h[1]
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[2]
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[2]
+; CHECK-CVT-SD-NEXT: mov h16, v1.h[3]
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s4, s5, s4
+; CHECK-CVT-SD-NEXT: shll v5.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v6.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: mov h7, v1.h[4]
+; CHECK-CVT-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s2, s3, s2
+; CHECK-CVT-SD-NEXT: mov h3, v0.h[3]
+; CHECK-CVT-SD-NEXT: fmov w9, s4
+; CHECK-CVT-SD-NEXT: mov h4, v0.h[4]
+; CHECK-CVT-SD-NEXT: fmaxnm s5, s6, s5
+; CHECK-CVT-SD-NEXT: shll v6.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w11, s2
+; CHECK-CVT-SD-NEXT: shll v2.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w9, w9, w8
+; CHECK-CVT-SD-NEXT: mov h3, v1.h[5]
+; CHECK-CVT-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w10, w9
+; CHECK-CVT-SD-NEXT: ubfx w12, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w10, w11, w8
+; CHECK-CVT-SD-NEXT: fmaxnm s7, s2, s16
+; CHECK-CVT-SD-NEXT: fmov w11, s5
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov h5, v0.h[5]
+; CHECK-CVT-SD-NEXT: add w10, w12, w10
+; CHECK-CVT-SD-NEXT: fmaxnm s4, s4, s6
+; CHECK-CVT-SD-NEXT: mov h6, v1.h[6]
+; CHECK-CVT-SD-NEXT: lsr w10, w10, #16
+; CHECK-CVT-SD-NEXT: fmov s2, w9
+; CHECK-CVT-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT: ubfx w9, w11, #16, #1
+; CHECK-CVT-SD-NEXT: add w11, w11, w8
+; CHECK-CVT-SD-NEXT: mov h1, v1.h[7]
+; CHECK-CVT-SD-NEXT: fmov s16, w10
+; CHECK-CVT-SD-NEXT: fmov w10, s7
+; CHECK-CVT-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w9, w11
+; CHECK-CVT-SD-NEXT: mov h7, v0.h[6]
+; CHECK-CVT-SD-NEXT: mov h0, v0.h[7]
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: mov v2.h[1], v16.h[0]
+; CHECK-CVT-SD-NEXT: add w10, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s16, w9
+; CHECK-CVT-SD-NEXT: fmaxnm s3, s5, s3
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: add w9, w11, w10
+; CHECK-CVT-SD-NEXT: fmov w10, s4
+; CHECK-CVT-SD-NEXT: shll v4.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT: shll v5.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[2], v16.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s6, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: fmaxnm s3, s5, s4
+; CHECK-CVT-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[3], v6.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s4, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s3
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[4], v4.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: fmov s1, w9
+; CHECK-CVT-SD-NEXT: add w9, w10, w8
+; CHECK-CVT-SD-NEXT: fmov w10, s0
+; CHECK-CVT-SD-NEXT: add w9, w11, w9
+; CHECK-CVT-SD-NEXT: lsr w9, w9, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[5], v1.h[0]
+; CHECK-CVT-SD-NEXT: ubfx w11, w10, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: fmov s0, w9
+; CHECK-CVT-SD-NEXT: add w8, w11, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: mov v2.h[6], v0.h[0]
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-CVT-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_maxnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: mov h2, v1.h[1]
-; CHECK-BF16-NEXT: mov h3, v0.h[1]
-; CHECK-BF16-NEXT: shll v4.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v0.4h, #16
-; CHECK-BF16-NEXT: mov h6, v1.h[2]
-; CHECK-BF16-NEXT: mov h7, v0.h[2]
-; CHECK-BF16-NEXT: mov h16, v1.h[3]
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s4, s5, s4
-; CHECK-BF16-NEXT: mov h5, v0.h[3]
-; CHECK-BF16-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s3, s3, s2
-; CHECK-BF16-NEXT: bfcvt h2, s4
-; CHECK-BF16-NEXT: fmaxnm s4, s7, s6
-; CHECK-BF16-NEXT: shll v6.4s, v16.4h, #16
-; CHECK-BF16-NEXT: shll v5.4s, v5.4h, #16
-; CHECK-BF16-NEXT: mov h7, v1.h[4]
-; CHECK-BF16-NEXT: mov h16, v0.h[4]
-; CHECK-BF16-NEXT: bfcvt h3, s3
-; CHECK-BF16-NEXT: fmaxnm s5, s5, s6
-; CHECK-BF16-NEXT: bfcvt h4, s4
-; CHECK-BF16-NEXT: mov h6, v0.h[5]
-; CHECK-BF16-NEXT: shll v7.4s, v7.4h, #16
-; CHECK-BF16-NEXT: shll v16.4s, v16.4h, #16
-; CHECK-BF16-NEXT: mov v2.h[1], v3.h[0]
-; CHECK-BF16-NEXT: mov h3, v1.h[5]
-; CHECK-BF16-NEXT: bfcvt h5, s5
-; CHECK-BF16-NEXT: fmaxnm s7, s16, s7
-; CHECK-BF16-NEXT: mov h16, v0.h[6]
-; CHECK-BF16-NEXT: shll v6.4s, v6.4h, #16
-; CHECK-BF16-NEXT: mov h0, v0.h[7]
-; CHECK-BF16-NEXT: mov v2.h[2], v4.h[0]
-; CHECK-BF16-NEXT: mov h4, v1.h[6]
-; CHECK-BF16-NEXT: shll v3.4s, v3.4h, #16
-; CHECK-BF16-NEXT: mov h1, v1.h[7]
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s3, s6, s3
-; CHECK-BF16-NEXT: shll v6.4s, v16.4h, #16
-; CHECK-BF16-NEXT: mov v2.h[3], v5.h[0]
-; CHECK-BF16-NEXT: bfcvt h5, s7
-; CHECK-BF16-NEXT: shll v4.4s, v4.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s4, s6, s4
-; CHECK-BF16-NEXT: bfcvt h3, s3
-; CHECK-BF16-NEXT: mov v2.h[4], v5.h[0]
-; CHECK-BF16-NEXT: fmaxnm s0, s0, s1
-; CHECK-BF16-NEXT: mov v2.h[5], v3.h[0]
-; CHECK-BF16-NEXT: bfcvt h3, s4
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: mov v2.h[6], v3.h[0]
-; CHECK-BF16-NEXT: mov v2.h[7], v0.h[0]
-; CHECK-BF16-NEXT: mov v0.16b, v2.16b
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_maxnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: mov h6, v1.h[2]
+; CHECK-BF16-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16-SD-NEXT: mov h16, v1.h[3]
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s4, s5, s4
+; CHECK-BF16-SD-NEXT: mov h5, v0.h[3]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s3, s3, s2
+; CHECK-BF16-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16-SD-NEXT: fmaxnm s4, s7, s6
+; CHECK-BF16-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT: mov h7, v1.h[4]
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[4]
+; CHECK-BF16-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16-SD-NEXT: fmaxnm s5, s5, s6
+; CHECK-BF16-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: mov v2.h[1], v3.h[0]
+; CHECK-BF16-SD-NEXT: mov h3, v1.h[5]
+; CHECK-BF16-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16-SD-NEXT: fmaxnm s7, s16, s7
+; CHECK-BF16-SD-NEXT: mov h16, v0.h[6]
+; CHECK-BF16-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16-SD-NEXT: mov v2.h[2], v4.h[0]
+; CHECK-BF16-SD-NEXT: mov h4, v1.h[6]
+; CHECK-BF16-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s3, s6, s3
+; CHECK-BF16-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16-SD-NEXT: mov v2.h[3], v5.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h5, s7
+; CHECK-BF16-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s4, s6, s4
+; CHECK-BF16-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16-SD-NEXT: mov v2.h[4], v5.h[0]
+; CHECK-BF16-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16-SD-NEXT: mov v2.h[5], v3.h[0]
+; CHECK-BF16-SD-NEXT: bfcvt h3, s4
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: mov v2.h[6], v3.h[0]
+; CHECK-BF16-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-BF16-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_maxnum:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: mov h2, v1.h[1]
+; CHECK-BF16SVE-SD-NEXT: mov h3, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h6, v1.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h7, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT: mov h16, v1.h[3]
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s4, s5, s4
+; CHECK-BF16SVE-SD-NEXT: mov h5, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s3, s3, s2
+; CHECK-BF16SVE-SD-NEXT: bfcvt h2, s4
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s4, s7, s6
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h7, v1.h[4]
+; CHECK-BF16SVE-SD-NEXT: mov h16, v0.h[4]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s5, s5, s6
+; CHECK-BF16SVE-SD-NEXT: bfcvt h4, s4
+; CHECK-BF16SVE-SD-NEXT: mov h6, v0.h[5]
+; CHECK-BF16SVE-SD-NEXT: shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v16.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[1], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h3, v1.h[5]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h5, s5
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s7, s16, s7
+; CHECK-BF16SVE-SD-NEXT: mov h16, v0.h[6]
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h0, v0.h[7]
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[2], v4.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov h4, v1.h[6]
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov h1, v1.h[7]
+; CHECK-BF16SVE-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s3, s6, s3
+; CHECK-BF16SVE-SD-NEXT: shll v6.4s, v16.4h, #16
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[3], v5.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h5, s7
+; CHECK-BF16SVE-SD-NEXT: shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s4, s6, s4
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s3
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[4], v5.h[0]
+; CHECK-BF16SVE-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[5], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: bfcvt h3, s4
+; CHECK-BF16SVE-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[6], v3.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v2.h[7], v0.h[0]
+; CHECK-BF16SVE-SD-NEXT: mov v0.16b, v2.16b
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_maxnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmaxnm v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fmaxnm v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_maxnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fmaxnm v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fmaxnm v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.maxnum.v8bf16(<8 x bfloat> %a, <8 x bfloat> %b)
ret <8 x bfloat> %r
}
@@ -6057,263 +6687,613 @@ define <8 x bfloat> @test_copysign_f32(<8 x bfloat> %a, <8 x float> %b) #0 {
}
define <8 x bfloat> @test_floor(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_floor:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frintm v2.4s, v2.4s
-; CHECK-CVT-NEXT: frintm v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_floor:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frintm v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frintm v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_floor:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frintm v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frintm v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_floor:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frintm v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frintm v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_floor:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frintm v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frintm v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_floor:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintm v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frintm v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_floor:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frintm v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frintm v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.floor.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_ceil(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_ceil:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frintp v2.4s, v2.4s
-; CHECK-CVT-NEXT: frintp v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_ceil:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frintp v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frintp v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_ceil:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frintp v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frintp v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_ceil:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frintp v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frintp v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_ceil:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frintp v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frintp v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_ceil:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintp v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frintp v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_ceil:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frintp v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frintp v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.ceil.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_trunc(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_trunc:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frintz v2.4s, v2.4s
-; CHECK-CVT-NEXT: frintz v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_trunc:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frintz v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frintz v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_trunc:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frintz v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frintz v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_trunc:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frintz v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frintz v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_trunc:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frintz v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frintz v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_trunc:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintz v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frintz v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_trunc:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frintz v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frintz v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.trunc.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_rint(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_rint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frintx v2.4s, v2.4s
-; CHECK-CVT-NEXT: frintx v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_rint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frintx v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frintx v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_rint:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frintx v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frintx v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_rint:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frintx v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frintx v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_rint:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frintx v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frintx v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_rint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintx v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frintx v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_rint:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frintx v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frintx v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.rint.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_nearbyint(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_nearbyint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frinti v2.4s, v2.4s
-; CHECK-CVT-NEXT: frinti v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_nearbyint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frinti v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frinti v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_nearbyint:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frinti v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frinti v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_nearbyint:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frinti v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frinti v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_nearbyint:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frinti v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frinti v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_nearbyint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frinti v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frinti v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_nearbyint:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frinti v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frinti v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.nearbyint.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_round(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_round:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frinta v2.4s, v2.4s
-; CHECK-CVT-NEXT: frinta v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_round:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frinta v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frinta v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_round:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frinta v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frinta v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_round:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frinta v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frinta v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_round:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frinta v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frinta v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_round:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frinta v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frinta v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_round:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frinta v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frinta v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.round.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_roundeven(<8 x bfloat> %a) #0 {
-; CHECK-CVT-LABEL: test_roundeven:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v2.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: frintn v2.4s, v2.4s
-; CHECK-CVT-NEXT: frintn v3.4s, v0.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v3.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b
-; CHECK-CVT-NEXT: and v1.16b, v4.16b, v1.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_roundeven:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: frintn v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT: frintn v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-CVT-SD-NEXT: and v1.16b, v4.16b, v1.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_roundeven:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: shll v1.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v2.4s, v0.8h, #16
-; CHECK-BF16-NEXT: frintn v1.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s
-; CHECK-BF16-NEXT: frintn v1.4s, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_roundeven:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: frintn v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT: frintn v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_roundeven:
+; CHECK-BF16SVE-SD: // %bb.0:
+; CHECK-BF16SVE-SD-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: frintn v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: frintn v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_roundeven:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #1
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: frintn v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: frintn v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v1.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v1.16b, v1.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v2.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v5.16b, v2.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-GI-NEXT: bif v1.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v2.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_roundeven:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: frintn v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT: frintn v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT: ret
%r = call <8 x bfloat> @llvm.roundeven.v8bf16(<8 x bfloat> %a)
ret <8 x bfloat> %r
}
define <8 x bfloat> @test_fmuladd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat> %c) #0 {
-; CHECK-CVT-LABEL: test_fmuladd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v6.4s, #127, msl #8
-; CHECK-CVT-NEXT: fmul v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: movi v4.4s, #1
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v1.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v1.16b, v4.16b
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: and v3.16b, v5.16b, v4.16b
-; CHECK-CVT-NEXT: addhn v1.4h, v1.4s, v6.4s
-; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s
-; CHECK-CVT-NEXT: shll v3.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll2 v2.4s, v2.8h, #16
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v6.4s
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: fadd v1.4s, v1.4s, v3.4s
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v2.4s, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v4.16b
-; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: and v1.16b, v3.16b, v4.16b
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v6.4s
-; CHECK-CVT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v6.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmuladd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v6.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: movi v4.4s, #1
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v1.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v1.16b, v4.16b
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: and v3.16b, v5.16b, v4.16b
+; CHECK-CVT-SD-NEXT: addhn v1.4h, v1.4s, v6.4s
+; CHECK-CVT-SD-NEXT: add v0.4s, v3.4s, v0.4s
+; CHECK-CVT-SD-NEXT: shll v3.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v6.4s
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v1.4s, v1.4s, v3.4s
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v2.4s, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v4.16b
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: and v1.16b, v3.16b, v4.16b
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v6.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v6.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-SD-LABEL: test_fmuladd:
; CHECK-BF16-SD: // %bb.0:
@@ -6354,13 +7334,69 @@ define <8 x bfloat> @test_fmuladd(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat>
; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
; CHECK-BF16SVE-SD-NEXT: ret
;
+; CHECK-CVT-GI-LABEL: test_fmuladd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v4.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v7.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: movi v4.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v16.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v17.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v6.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v18.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v19.4s, v0.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: and v5.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v6.16b, v6.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v16.16b, v16.16b
+; CHECK-CVT-GI-NEXT: add v5.4s, v17.4s, v5.4s
+; CHECK-CVT-GI-NEXT: mvn v17.16b, v18.16b
+; CHECK-CVT-GI-NEXT: add v6.4s, v19.4s, v6.4s
+; CHECK-CVT-GI-NEXT: bif v3.16b, v5.16b, v16.16b
+; CHECK-CVT-GI-NEXT: shll v5.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v2.4s, v2.8h, #16
+; CHECK-CVT-GI-NEXT: bif v0.16b, v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: shrn v3.4h, v3.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v3.4h, #16
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd v3.4s, v3.4s, v5.4s
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: ushr v2.4s, v3.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v6.4s, v3.4s, v3.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v3.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v4.4s, v0.4s, v4.4s
+; CHECK-CVT-GI-NEXT: orr v3.16b, v3.16b, v7.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: and v2.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v6.16b
+; CHECK-CVT-GI-NEXT: mvn v6.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v2.4s, v16.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v4.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bit v2.16b, v3.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
; CHECK-BF16-GI-LABEL: test_fmuladd:
; CHECK-BF16-GI: // %bb.0:
-; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-BF16-GI-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v4.4s, v1.4h, #16
; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-GI-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fmul v3.4s, v3.4s, v4.4s
; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-BF16-GI-NEXT: bfcvtn v1.4h, v3.4s
; CHECK-BF16-GI-NEXT: shll v3.4s, v2.4h, #16
More information about the llvm-commits
mailing list