[llvm] [AArch64][GlobalISel] Enable BF16 legalization for fadd and friends. (PR #196081)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 07:05:57 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/196081
This enabled bf16 promotion for the following operations in GISel, promoting them to f32 and truncating the result back:
G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM, G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN
>From e151c7f27c8ed0860fee5ddda85e7e7d83663836 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 6 May 2026 14:58:30 +0100
Subject: [PATCH] [AArch64][GlobalISel] Enable BF16 legalization for fadd and
friends.
This enabled bf16 promotion for the following operations in GISel, promoting
them to f32 and truncating the result back:
G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +-
.../AArch64/Atomics/aarch64-atomicrmw-lsfe.ll | 285 ++--
.../Atomics/aarch64-atomicrmw-v8a_fp.ll | 660 ++++++---
.../GlobalISel/legalizer-info-validation.mir | 68 +-
.../test/CodeGen/AArch64/bf16-instructions.ll | 1295 +++++++++++------
.../CodeGen/AArch64/bf16-v4-instructions.ll | 272 ++--
.../CodeGen/AArch64/bf16-v8-instructions.ll | 438 ++++--
7 files changed, 2014 insertions(+), 1011 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e8bc74a150b57..9771d167dbba7 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -450,7 +450,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor(HasFP16, {f16, v4f16, v8f16})
.libcallFor({f128})
.scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
- .minScalarOrElt(0, MinFPScalar)
+ .widenScalarIf(
+ [=](const LegalityQuery &Q) {
+ return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
+ Q.Types[0].getScalarType().isBFloat16();
+ },
+ changeElementTo(0, f32))
.clampNumElements(0, v4s16, v8s16)
.clampNumElements(0, v2s32, v4s32)
.clampNumElements(0, v2s64, v2s64)
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
index 290c8a60d700d..6a0b689b16888 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lsfe.ll
@@ -239,8 +239,11 @@ define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
@@ -253,8 +256,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
@@ -267,8 +273,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
@@ -281,8 +290,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
@@ -295,8 +307,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
@@ -449,12 +464,15 @@ define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -467,12 +485,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -485,12 +506,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -503,12 +527,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -521,12 +548,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -714,8 +744,11 @@ define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
@@ -728,8 +761,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
@@ -742,8 +778,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
@@ -756,8 +795,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
@@ -770,8 +812,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
@@ -1085,8 +1130,11 @@ define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
@@ -1099,8 +1147,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
@@ -1113,8 +1164,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
@@ -1127,8 +1181,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
@@ -1141,8 +1198,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
@@ -1456,8 +1516,11 @@ define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
@@ -1470,8 +1533,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
@@ -1484,8 +1550,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
@@ -1498,8 +1567,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
@@ -1512,8 +1584,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
@@ -1827,8 +1902,11 @@ define dso_local half @atomicrmw_fmaximum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
@@ -1841,8 +1919,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
@@ -1855,8 +1936,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
@@ -1869,8 +1953,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
@@ -1883,8 +1970,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
@@ -2198,8 +2288,11 @@ define dso_local half @atomicrmw_fminimum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
@@ -2212,8 +2305,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
@@ -2226,8 +2322,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
@@ -2240,8 +2339,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
@@ -2254,8 +2356,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
diff --git a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
index ccb4fd61b002c..d71231059c3b3 100644
--- a/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
+++ b/llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a_fp.ll
@@ -75,12 +75,15 @@ define dso_local half @atomicrmw_fadd_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -93,12 +96,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -111,12 +117,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -129,12 +138,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -147,12 +159,15 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fadd_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fadd_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -340,8 +355,11 @@ define dso_local half @atomicrmw_fadd_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_monotonic:
@@ -354,8 +372,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acquire:
@@ -368,8 +389,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_release:
@@ -382,8 +406,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_acq_rel:
@@ -396,8 +423,11 @@ define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fadd_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fadd_bfloat_unaligned_seq_cst:
@@ -550,12 +580,15 @@ define dso_local half @atomicrmw_fsub_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -568,12 +601,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -586,12 +622,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -604,12 +643,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -622,12 +664,15 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fsub_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fsub_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -815,8 +860,11 @@ define dso_local half @atomicrmw_fsub_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_monotonic:
@@ -829,8 +877,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acquire:
@@ -843,8 +894,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_release:
@@ -857,8 +911,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_acq_rel:
@@ -871,8 +928,11 @@ define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fsub_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fsub_bfloat_unaligned_seq_cst:
@@ -1025,12 +1085,15 @@ define dso_local half @atomicrmw_fmax_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1043,12 +1106,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -1061,12 +1127,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -1079,12 +1148,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -1097,12 +1169,15 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmax_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmax_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -1290,8 +1365,11 @@ define dso_local half @atomicrmw_fmax_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_monotonic:
@@ -1304,8 +1382,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acquire:
@@ -1318,8 +1399,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_release:
@@ -1332,8 +1416,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_acq_rel:
@@ -1346,8 +1433,11 @@ define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmax_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmax_bfloat_unaligned_seq_cst:
@@ -1500,12 +1590,15 @@ define dso_local half @atomicrmw_fmin_half_aligned_seq_cst(ptr %ptr, half %value
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1518,12 +1611,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_monotonic(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -1536,12 +1632,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acquire(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -1554,12 +1653,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_release(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -1572,12 +1674,15 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_acq_rel(ptr %ptr, bfloat
define dso_local bfloat @atomicrmw_fmin_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmin_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -1765,8 +1870,11 @@ define dso_local half @atomicrmw_fmin_half_unaligned_seq_cst(ptr %ptr, half %val
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_monotonic:
@@ -1779,8 +1887,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_monotonic(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acquire:
@@ -1793,8 +1904,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acquire(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_release:
@@ -1807,8 +1921,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_release(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_acq_rel:
@@ -1821,8 +1938,11 @@ define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_acq_rel(ptr %ptr, bfloa
define dso_local bfloat @atomicrmw_fmin_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmin_bfloat_unaligned_seq_cst:
@@ -1975,12 +2095,15 @@ define dso_local half @atomicrmw_fmaximum_half_aligned_seq_cst(ptr %ptr, half %v
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -1993,12 +2116,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_monotonic(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -2011,12 +2137,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acquire(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -2029,12 +2158,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_release(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -2047,12 +2179,15 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_acq_rel(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fmaximum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -2240,8 +2375,11 @@ define dso_local half @atomicrmw_fmaximum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_monotonic:
@@ -2254,8 +2392,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acquire:
@@ -2268,8 +2409,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_release:
@@ -2282,8 +2426,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_acq_rel:
@@ -2296,8 +2443,11 @@ define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fmaximum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fmaximum_bfloat_unaligned_seq_cst:
@@ -2450,12 +2600,15 @@ define dso_local half @atomicrmw_fminimum_half_aligned_seq_cst(ptr %ptr, half %v
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_monotonic:
; -O1: ldxrh w9, [x0]
@@ -2468,12 +2621,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_monotonic(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acquire:
; -O1: ldaxrh w9, [x0]
@@ -2486,12 +2642,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acquire(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_release:
; -O1: ldxrh w9, [x0]
@@ -2504,12 +2663,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_release(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_acq_rel:
; -O1: ldaxrh w9, [x0]
@@ -2522,12 +2684,15 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_acq_rel(ptr %ptr, bfl
define dso_local bfloat @atomicrmw_fminimum_bfloat_aligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
-; -O0: ldaxrh w8, [x11]
-; -O0: cmp w8, w9, uxth
-; -O0: stlxrh w10, w12, [x11]
-; -O0: subs w9, w8, w9, uxth
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w11, w9, w11
+; -O0: add w9, w9, w11
+; -O0: csel w8, w8, w9, vs
+; -O0: ldaxrh w9, [x10]
+; -O0: cmp w9, w11, uxth
+; -O0: stlxrh w8, w12, [x10]
+; -O0: subs w8, w8, w9, uxth
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_aligned_seq_cst:
; -O1: ldaxrh w9, [x0]
@@ -2715,8 +2880,11 @@ define dso_local half @atomicrmw_fminimum_half_unaligned_seq_cst(ptr %ptr, half
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_monotonic:
@@ -2729,8 +2897,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_monotonic(ptr %ptr,
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acquire:
@@ -2743,8 +2914,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acquire(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_release:
@@ -2757,8 +2931,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_release(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_acq_rel:
@@ -2771,8 +2948,11 @@ define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_acq_rel(ptr %ptr, b
define dso_local bfloat @atomicrmw_fminimum_bfloat_unaligned_seq_cst(ptr %ptr, bfloat %value) {
; -O0-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
-; -O0: add w8, w8, w9
-; -O0: add w8, w8, w9
+; -O0: orr w8, w8, #0x400000
+; -O0: and w9, w9, #0x1
+; -O0: add w10, w9, w10
+; -O0: add w9, w9, w10
+; -O0: csel w8, w8, w9, vs
; -O0: bl __atomic_compare_exchange
;
; -O1-LABEL: atomicrmw_fminimum_bfloat_unaligned_seq_cst:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index ff4fbc0e1afeb..17bb373070568 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -179,12 +179,12 @@
#
# DEBUG-NEXT: G_INTRINSIC_TRUNC (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_ROUND (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
@@ -195,8 +195,8 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_READCYCLECOUNTER (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -516,27 +516,27 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FADD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FSUB (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMUL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMA (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAD (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FDIV (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FREM (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -627,12 +627,12 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMINNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAXNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMINNUM_IEEE (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -641,12 +641,12 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: G_FMINIMUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMAXIMUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FMINIMUMNUM (opcode {{[0-9]+}}): 1 type index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
@@ -772,8 +772,8 @@
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FCEIL (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FCOS (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -818,20 +818,20 @@
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
# DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FFLOOR (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FRINT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_FNEARBYINT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 1, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_ADDRSPACE_CAST (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index b3b9653b14481..0e0c768bf4878 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -4,12 +4,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc_f32_f16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
@@ -40,7 +35,6 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fptrunc_double
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -58,26 +52,10 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_copysign_f64
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
-;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_fadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fsub
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmul
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmadd
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fdiv
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_frem
+;
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for test_frem
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_select_cc_f32_f16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_une
@@ -108,7 +86,6 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sitofp_i32_fadd
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fptrunc_double
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fpext_double
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sqrt
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_powi
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_sin
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_cos
@@ -126,184 +103,342 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log10
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_log2
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fma
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fabs
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_minnum
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_maxnum
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_copysign_f64
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_floor
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_ceil
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_trunc
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_rint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_nearbyint
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_round
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_roundeven
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fmuladd
+;
define bfloat @test_fadd(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fadd bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fsub(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fsub:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsub s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fsub:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsub s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fsub:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fsub s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fsub:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fsub s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fsub:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fsub s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fsub:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fsub s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fsub bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fmul(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fmul:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmul:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmul:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmul:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmul:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmul:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fmul bfloat %a, %b
ret bfloat %r
}
define bfloat @test_fmadd(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fmadd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmadd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmadd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: fmadd s0, s0, s1, s2
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %mul = fmul fast bfloat %a, %b
- %r = fadd fast bfloat %mul, %c
- ret bfloat %r
-}
-
-define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_fdiv:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fdiv s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmadd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fdiv:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fdiv s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-CVT-GI-LABEL: test_fmadd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w11, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmadd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %mul = fmul fast bfloat %a, %b
+ %r = fadd fast bfloat %mul, %c
+ ret bfloat %r
+}
+
+define bfloat @test_fdiv(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fdiv:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fdiv s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_fdiv:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fdiv s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fdiv:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fdiv s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fdiv:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fdiv s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = fdiv bfloat %a, %b
ret bfloat %r
}
@@ -1182,20 +1317,20 @@ define bfloat @test_bitcast_i16tobfloat(i16 %a) #0 {
}
define bfloat @test_sqrt(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_sqrt:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsqrt s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_sqrt:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsqrt s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_sqrt:
; CHECK-BF16: // %bb.0:
@@ -1204,6 +1339,24 @@ define bfloat @test_sqrt(bfloat %a) #0 {
; CHECK-BF16-NEXT: fsqrt s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_sqrt:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fsqrt s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.sqrt.bf16(bfloat %a)
ret bfloat %r
}
@@ -1782,36 +1935,70 @@ define bfloat @test_log2(bfloat %a) #0 {
}
define bfloat @test_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fma:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmadd s0, s0, s1, s2
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fma:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fma:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v2.4s, v2.4h, #16
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmadd s0, s0, s1, s2
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fma:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fma:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmadd s0, s0, s1, s2
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fma:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll v2.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: fmadd s0, s0, s1, s2
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %r
}
@@ -1830,103 +2017,163 @@ define bfloat @test_fabs(bfloat %a) #0 {
}
define bfloat @test_minnum(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_minnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fminnm s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
-;
-; CHECK-BF16-LABEL: test_minnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fminnm s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %r = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
- ret bfloat %r
-}
-
-define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-LABEL: test_maxnum:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmaxnm s0, s0, s1
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
-;
-; CHECK-BF16-LABEL: test_maxnum:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmaxnm s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
- %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
- ret bfloat %r
-}
-
-define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
-; CHECK-CVT-SD-LABEL: test_copysign:
+; CHECK-CVT-SD-LABEL: test_minnum:
; CHECK-CVT-SD: // %bb.0:
; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v2.16b
-; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: fminnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
; CHECK-CVT-SD-NEXT: fmov s0, w8
; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-SD-LABEL: test_copysign:
+; CHECK-BF16-SD-LABEL: test_minnum:
; CHECK-BF16-SD: // %bb.0:
; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-SD-NEXT: mvni v2.4s, #128, lsl #24
; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-SD-NEXT: fminnm s0, s0, s1
; CHECK-BF16-SD-NEXT: bfcvt h0, s0
; CHECK-BF16-SD-NEXT: ret
;
-; CHECK-CVT-GI-LABEL: test_copysign:
+; CHECK-CVT-GI-LABEL: test_minnum:
; CHECK-CVT-GI: // %bb.0:
-; CHECK-CVT-GI-NEXT: mvni v2.4h, #128, lsl #8
; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-GI-NEXT: bif v0.8b, v1.8b, v2.8b
-; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
-; CHECK-CVT-GI-NEXT: ret
-;
-; CHECK-BF16-GI-LABEL: test_copysign:
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fminnm s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_minnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fminnm s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %r = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_maxnum:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_maxnum:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_maxnum:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmaxnm s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_maxnum:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmaxnm s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
+ %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+ ret bfloat %r
+}
+
+define bfloat @test_copysign(bfloat %a, bfloat %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
+;
+; CHECK-BF16-SD-LABEL: test_copysign:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: mvni v2.4s, #128, lsl #24
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_copysign:
; CHECK-BF16-GI: // %bb.0:
; CHECK-BF16-GI-NEXT: mvni v2.4h, #128, lsl #8
; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
@@ -2075,20 +2322,20 @@ define float @test_copysign_extended(bfloat %a, bfloat %b) #0 {
}
define bfloat @test_floor(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_floor:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintm s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_floor:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintm s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_floor:
; CHECK-BF16: // %bb.0:
@@ -2097,25 +2344,43 @@ define bfloat @test_floor(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintm s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_floor:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintm s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.floor.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_ceil(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_ceil:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintp s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_ceil:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintp s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_ceil:
; CHECK-BF16: // %bb.0:
@@ -2124,25 +2389,43 @@ define bfloat @test_ceil(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintp s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_ceil:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintp s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.ceil.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_trunc(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_trunc:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintz s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_trunc:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintz s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_trunc:
; CHECK-BF16: // %bb.0:
@@ -2151,25 +2434,43 @@ define bfloat @test_trunc(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintz s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_trunc:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintz s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.trunc.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_rint(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_rint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintx s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_rint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintx s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_rint:
; CHECK-BF16: // %bb.0:
@@ -2178,25 +2479,43 @@ define bfloat @test_rint(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintx s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_rint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintx s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.rint.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_nearbyint(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_nearbyint:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frinti s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_nearbyint:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frinti s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_nearbyint:
; CHECK-BF16: // %bb.0:
@@ -2205,25 +2524,43 @@ define bfloat @test_nearbyint(bfloat %a) #0 {
; CHECK-BF16-NEXT: frinti s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_nearbyint:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frinti s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.nearbyint.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_round(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_round:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frinta s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_round:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frinta s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_round:
; CHECK-BF16: // %bb.0:
@@ -2232,25 +2569,43 @@ define bfloat @test_round(bfloat %a) #0 {
; CHECK-BF16-NEXT: frinta s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_round:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frinta s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_roundeven(bfloat %a) #0 {
-; CHECK-CVT-LABEL: test_roundeven:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w8, #32767 // =0x7fff
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: frintn s0, s0
-; CHECK-CVT-NEXT: fmov w9, s0
-; CHECK-CVT-NEXT: ubfx w10, w9, #16, #1
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: add w8, w10, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_roundeven:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: frintn s0, s0
+; CHECK-CVT-SD-NEXT: fmov w9, s0
+; CHECK-CVT-SD-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: add w8, w10, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-LABEL: test_roundeven:
; CHECK-BF16: // %bb.0:
@@ -2259,52 +2614,118 @@ define bfloat @test_roundeven(bfloat %a) #0 {
; CHECK-BF16-NEXT: frintn s0, s0
; CHECK-BF16-NEXT: bfcvt h0, s0
; CHECK-BF16-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_roundeven:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: frintn s0, s0
+; CHECK-CVT-GI-NEXT: fmov w9, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w10, w9, #16, #1
+; CHECK-CVT-GI-NEXT: add w8, w9, w8
+; CHECK-CVT-GI-NEXT: orr w9, w9, #0x400000
+; CHECK-CVT-GI-NEXT: add w8, w8, w10
+; CHECK-CVT-GI-NEXT: csel w8, w9, w8, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
%r = call bfloat @llvm.roundeven.bf16(bfloat %a)
ret bfloat %r
}
define bfloat @test_fmuladd(bfloat %a, bfloat %b, bfloat %c) #0 {
-; CHECK-CVT-LABEL: test_fmuladd:
-; CHECK-CVT: // %bb.0:
-; CHECK-CVT-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-CVT-NEXT: mov w10, #32767 // =0x7fff
-; CHECK-CVT-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: ubfx w9, w8, #16, #1
-; CHECK-CVT-NEXT: add w8, w8, w10
-; CHECK-CVT-NEXT: add w8, w9, w8
-; CHECK-CVT-NEXT: lsr w8, w8, #16
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: // kill: def $h0 killed $h0 killed $s0
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: test_fmuladd:
+; CHECK-CVT-SD: // %bb.0:
+; CHECK-CVT-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-SD-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT: fmov w8, s0
+; CHECK-CVT-SD-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-SD-NEXT: add w8, w8, w10
+; CHECK-CVT-SD-NEXT: add w8, w9, w8
+; CHECK-CVT-SD-NEXT: lsr w8, w8, #16
+; CHECK-CVT-SD-NEXT: fmov s0, w8
+; CHECK-CVT-SD-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: test_fmuladd:
-; CHECK-BF16: // %bb.0:
-; CHECK-BF16-NEXT: // kill: def $h1 killed $h1 def $d1
-; CHECK-BF16-NEXT: // kill: def $h0 killed $h0 def $d0
-; CHECK-BF16-NEXT: // kill: def $h2 killed $h2 def $d2
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul s0, s0, s1
-; CHECK-BF16-NEXT: shll v1.4s, v2.4h, #16
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd s0, s0, s1
-; CHECK-BF16-NEXT: bfcvt h0, s0
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: test_fmuladd:
+; CHECK-BF16-SD: // %bb.0:
+; CHECK-BF16-SD-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-SD-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-SD-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul s0, s0, s1
+; CHECK-BF16-SD-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd s0, s0, s1
+; CHECK-BF16-SD-NEXT: bfcvt h0, s0
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: test_fmuladd:
+; CHECK-CVT-GI: // %bb.0:
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT: mov w10, #32767 // =0x7fff
+; CHECK-CVT-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w11, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w11, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT: fmov w8, s0
+; CHECK-CVT-GI-NEXT: fcmp s0, #0.0
+; CHECK-CVT-GI-NEXT: ubfx w9, w8, #16, #1
+; CHECK-CVT-GI-NEXT: add w10, w8, w10
+; CHECK-CVT-GI-NEXT: orr w8, w8, #0x400000
+; CHECK-CVT-GI-NEXT: add w9, w10, w9
+; CHECK-CVT-GI-NEXT: csel w8, w8, w9, vs
+; CHECK-CVT-GI-NEXT: lsr w8, w8, #16
+; CHECK-CVT-GI-NEXT: fmov s0, w8
+; CHECK-CVT-GI-NEXT: // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: test_fmuladd:
+; CHECK-BF16-GI: // %bb.0:
+; CHECK-BF16-GI-NEXT: // kill: def $h0 killed $h0 def $d0
+; CHECK-BF16-GI-NEXT: // kill: def $h1 killed $h1 def $d1
+; CHECK-BF16-GI-NEXT: // kill: def $h2 killed $h2 def $d2
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul s0, s0, s1
+; CHECK-BF16-GI-NEXT: shll v1.4s, v2.4h, #16
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: fadd s0, s0, s1
+; CHECK-BF16-GI-NEXT: bfcvt h0, s0
+; CHECK-BF16-GI-NEXT: ret
%r = call bfloat @llvm.fmuladd.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %r
}
diff --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index b292a3de4f9af..1d85f3028c23e 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -4,11 +4,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for add_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sub_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for mul_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for d_to_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i16
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i32
@@ -36,11 +32,7 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ole
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ord
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for add_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sub_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for mul_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for d_to_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i16
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_i32
@@ -69,26 +61,53 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ord
define <4 x bfloat> @add_h(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: add_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: add_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: add_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fadd v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: add_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: add_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: add_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fadd <4 x bfloat> %a, %b
ret <4 x bfloat> %0
@@ -123,78 +142,159 @@ entry:
}
define <4 x bfloat> @sub_h(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: sub_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fsub v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: sub_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: sub_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fsub v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: sub_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: sub_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: sub_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fsub <4 x bfloat> %a, %b
ret <4 x bfloat> %0
}
define <4 x bfloat> @mul_h(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: mul_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: mul_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: mul_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fmul v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: mul_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: mul_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: mul_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fmul <4 x bfloat> %a, %b
ret <4 x bfloat> %0
}
define <4 x bfloat> @div_h(<4 x bfloat> %a, <4 x bfloat> %b) {
-; CHECK-CVT-LABEL: div_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-CVT-NEXT: fdiv v0.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v1.4s, #1
-; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16
-; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b
-; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8
-; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: div_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v1.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v2.4s, v0.4s, #16
+; CHECK-CVT-SD-NEXT: and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: div_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v1.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v0.4s, v0.4h, #16
-; CHECK-BF16-NEXT: fdiv v0.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: div_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: div_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: movi v5.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v3.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v4.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v2.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v5.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v3.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v3.16b, v4.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: div_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v1.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fdiv <4 x bfloat> %a, %b
ret <4 x bfloat> %0
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 626d54acdb6b8..12cf912794071 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -5,11 +5,7 @@
; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
-; CHECK-CVT-GI: warning: Instruction selection used fallback path for add_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sub_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for mul_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
+; CHECK-CVT-GI: warning: Instruction selection used fallback path for d_to_h
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v4i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v8i8
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v16i8
@@ -41,11 +37,7 @@
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ole
; CHECK-CVT-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ord
;
-; CHECK-BF16-GI: warning: Instruction selection used fallback path for add_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sub_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for mul_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for div_h
-; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for d_to_h
+; CHECK-BF16-GI: warning: Instruction selection used fallback path for d_to_h
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v4i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v8i8
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for sitofp_v16i8
@@ -78,37 +70,93 @@
; CHECK-BF16-GI-NEXT: warning: Instruction selection used fallback path for test_fcmp_ord
define <8 x bfloat> @add_h(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: add_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fadd v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fadd v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: add_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fadd v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: add_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fadd v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: add_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fadd v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: add_h:
+; CHECK-BF16SVE-SD: // %bb.0: // %entry
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fadd v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: add_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fadd v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fadd v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: add_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fadd v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fadd <8 x bfloat> %a, %b
ret <8 x bfloat> %0
@@ -149,62 +197,118 @@ entry:
}
define <8 x bfloat> @sub_h(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: sub_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fsub v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fsub v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: sub_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fsub v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: sub_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fsub v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fsub v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: sub_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fsub v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: sub_h:
+; CHECK-BF16SVE-SD: // %bb.0: // %entry
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fsub v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: sub_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fsub v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fsub v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: sub_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fsub v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fsub <8 x bfloat> %a, %b
ret <8 x bfloat> %0
}
define <8 x bfloat> @mul_h(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: mul_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v3.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v4.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v2.4s, #1
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fmul v3.4s, v4.4s, v3.4s
-; CHECK-CVT-NEXT: fmul v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: ushr v0.4s, v3.4s, #16
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v0.16b, v0.16b, v2.16b
-; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b
-; CHECK-CVT-NEXT: add v0.4s, v0.4s, v3.4s
-; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s
-; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: mul_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v4.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v2.4s, #1
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-CVT-SD-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: ushr v0.4s, v3.4s, #16
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v0.16b, v0.16b, v2.16b
+; CHECK-CVT-SD-NEXT: and v2.16b, v4.16b, v2.16b
+; CHECK-CVT-SD-NEXT: add v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v0.4s, v5.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
; CHECK-BF16-SD-LABEL: mul_h:
; CHECK-BF16-SD: // %bb.0: // %entry
@@ -230,13 +334,45 @@ define <8 x bfloat> @mul_h(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-BF16SVE-SD-NEXT: mov v0.16b, v2.16b
; CHECK-BF16SVE-SD-NEXT: ret
;
+; CHECK-CVT-GI-LABEL: mul_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fmul v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
; CHECK-BF16-GI-LABEL: mul_h:
; CHECK-BF16-GI: // %bb.0: // %entry
-; CHECK-BF16-GI-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-GI-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-GI-NEXT: fmul v2.4s, v3.4s, v2.4s
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fmul v2.4s, v2.4s, v3.4s
; CHECK-BF16-GI-NEXT: fmul v1.4s, v0.4s, v1.4s
; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
@@ -247,37 +383,93 @@ entry:
}
define <8 x bfloat> @div_h(<8 x bfloat> %a, <8 x bfloat> %b) {
-; CHECK-CVT-LABEL: div_h:
-; CHECK-CVT: // %bb.0: // %entry
-; CHECK-CVT-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-CVT-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-CVT-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-CVT-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8
-; CHECK-CVT-NEXT: fdiv v2.4s, v3.4s, v2.4s
-; CHECK-CVT-NEXT: fdiv v1.4s, v0.4s, v1.4s
-; CHECK-CVT-NEXT: movi v0.4s, #1
-; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v3.16b, v0.16b
-; CHECK-CVT-NEXT: add v2.4s, v3.4s, v2.4s
-; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16
-; CHECK-CVT-NEXT: and v3.16b, v4.16b, v0.16b
-; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v5.4s
-; CHECK-CVT-NEXT: add v1.4s, v3.4s, v1.4s
-; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s
-; CHECK-CVT-NEXT: ret
+; CHECK-CVT-SD-LABEL: div_h:
+; CHECK-CVT-SD: // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT: movi v5.4s, #127, msl #8
+; CHECK-CVT-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-CVT-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT: movi v0.4s, #1
+; CHECK-CVT-SD-NEXT: ushr v3.4s, v2.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v3.16b, v0.16b
+; CHECK-CVT-SD-NEXT: add v2.4s, v3.4s, v2.4s
+; CHECK-CVT-SD-NEXT: ushr v4.4s, v1.4s, #16
+; CHECK-CVT-SD-NEXT: and v3.16b, v4.16b, v0.16b
+; CHECK-CVT-SD-NEXT: addhn v0.4h, v2.4s, v5.4s
+; CHECK-CVT-SD-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-SD-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; CHECK-CVT-SD-NEXT: ret
;
-; CHECK-BF16-LABEL: div_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fdiv v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fdiv v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-BF16-SD-LABEL: div_h:
+; CHECK-BF16-SD: // %bb.0: // %entry
+; CHECK-BF16-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-BF16-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT: ret
+;
+; CHECK-BF16SVE-SD-LABEL: div_h:
+; CHECK-BF16SVE-SD: // %bb.0: // %entry
+; CHECK-BF16SVE-SD-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16SVE-SD-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT: fdiv v2.4s, v3.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16SVE-SD-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT: ret
+;
+; CHECK-CVT-GI-LABEL: div_h:
+; CHECK-CVT-GI: // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-CVT-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-CVT-GI-NEXT: movi v6.4s, #64, lsl #16
+; CHECK-CVT-GI-NEXT: fdiv v2.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: movi v3.4s, #127, msl #8
+; CHECK-CVT-GI-NEXT: fdiv v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT: movi v1.4s, #1
+; CHECK-CVT-GI-NEXT: ushr v4.4s, v2.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v7.4s, v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT: add v16.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v2.16b, v2.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v4.16b, v4.16b, v1.16b
+; CHECK-CVT-GI-NEXT: add v4.4s, v16.4s, v4.4s
+; CHECK-CVT-GI-NEXT: ushr v5.4s, v0.4s, #16
+; CHECK-CVT-GI-NEXT: fcmeq v17.4s, v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT: orr v0.16b, v0.16b, v6.16b
+; CHECK-CVT-GI-NEXT: and v1.16b, v5.16b, v1.16b
+; CHECK-CVT-GI-NEXT: mvn v5.16b, v7.16b
+; CHECK-CVT-GI-NEXT: mvn v7.16b, v17.16b
+; CHECK-CVT-GI-NEXT: add v1.4s, v3.4s, v1.4s
+; CHECK-CVT-GI-NEXT: bif v2.16b, v4.16b, v5.16b
+; CHECK-CVT-GI-NEXT: bit v1.16b, v0.16b, v7.16b
+; CHECK-CVT-GI-NEXT: shrn v0.4h, v2.4s, #16
+; CHECK-CVT-GI-NEXT: shrn v1.4h, v1.4s, #16
+; CHECK-CVT-GI-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-CVT-GI-NEXT: ret
+;
+; CHECK-BF16-GI-LABEL: div_h:
+; CHECK-BF16-GI: // %bb.0: // %entry
+; CHECK-BF16-GI-NEXT: shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT: shll v3.4s, v1.4h, #16
+; CHECK-BF16-GI-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-BF16-GI-NEXT: fdiv v2.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT: fdiv v1.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-BF16-GI-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-BF16-GI-NEXT: ret
entry:
%0 = fdiv <8 x bfloat> %a, %b
ret <8 x bfloat> %0
More information about the llvm-commits
mailing list