[llvm] [LLVM][CodeGen][SVE] Improve isel for split vector bfloat conversions. (PR #184357)
Paul Walker via llvm-commits
llvm-commits at lists.llvm.org
Tue Mar 3 07:07:32 PST 2026
https://github.com/paulwalker-arm created https://github.com/llvm/llvm-project/pull/184357
We typically promote nxv8bf16 operations to pairs of nxv4f32 operations, which leads to the common idiom:
fpextend(extract_[lo,hi] A)
This patch adds isel patterns to match the whole sequence, which is effectively just an interleave with zero.
>From a436a15cc8d35887fc69274123856fd4c37040e8 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker at arm.com>
Date: Tue, 3 Mar 2026 13:18:05 +0000
Subject: [PATCH] [LLVM][CodeGen][SVE] Improve isel for split vector bfloat
conversions.
We typically promote nxv8bf16 operations to pairs of nxv4f32
operations, which leads to the common idiom:
fpextend(extract_[lo,hi] A)
This patch adds isel patterns to match the whole sequence, which is
effectively just an interleave with zero.
---
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 12 +-
llvm/test/CodeGen/AArch64/sve-bf16-arith.ll | 137 ++++----
.../test/CodeGen/AArch64/sve-bf16-combines.ll | 279 +++++++---------
.../test/CodeGen/AArch64/sve-bf16-compares.ll | 304 +++++++-----------
.../test/CodeGen/AArch64/sve-bf16-converts.ll | 8 +-
.../CodeGen/AArch64/sve-bf16-int-converts.ll | 94 +++---
.../CodeGen/AArch64/sve-bf16-reductions.ll | 45 ++-
.../test/CodeGen/AArch64/sve-bf16-rounding.ll | 77 ++---
8 files changed, 409 insertions(+), 547 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index b1b1203ab512f..d4b6a97f168b2 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -389,8 +389,10 @@ def AArch64fdot : PatFrags<(ops node:$Zd, node:$Zn, node:$Zm),
]>;
def SDT_AArch64FCVT : SDTypeProfile<1, 3, [
- SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,
- SDTCVecEltisVT<1,i1>, SDTCisSameNumEltsAs<0,1>, SDTCisSameAs<0,3>
+ SDTCisVec<0>,
+ SDTCisVec<1>, SDTCisSameNumEltsAs<1,0>, SDTCVecEltisVT<1,i1>,
+ SDTCisVec<2>, SDTCisSameNumEltsAs<2,0>,
+ SDTCisSameAs<3,0>
]>;
def SDT_AArch64FCVTR : SDTypeProfile<1, 4, [
@@ -2519,6 +2521,12 @@ let Predicates = [HasSVE_or_SME] in {
def : Pat<(nxv2f32 (AArch64fcvte_mt (SVEAnyPredicate), nxv2bf16:$op, undef)),
(LSL_ZZI_S $op, (i32 16))>;
+ // A common sequence that occurs when promoting nxv8bf16 operations.
+ def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 0)), undef)),
+ (ZIP1_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>;
+ def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 4)), undef)),
+ (ZIP2_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>;
+
// Signed integer -> Floating-point
def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg),
(sext_inreg nxv2i64:$Zs, nxv2i16), nxv2f16:$Zd)),
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
index 6917ac12999bf..c916f81edf669 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
@@ -85,16 +85,13 @@ define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fadd_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fadd z2.s, z3.s, z2.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fadd z2.s, z4.s, z3.s
; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s
; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
@@ -142,18 +139,15 @@ define <vscale x 4 x bfloat> @fdiv_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fdiv_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fdiv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fdivr z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fdivr z3.s, p0/m, z3.s, z4.s
; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z3.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -206,18 +200,15 @@ define <vscale x 4 x bfloat> @fmax_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fmax_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmax_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fmax z2.s, p0/m, z2.s, z3.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fmax z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
+; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT: ret
@@ -276,18 +267,15 @@ define <vscale x 4 x bfloat> @fmaxnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale
define <vscale x 8 x bfloat> @fmaxnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmaxnm_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fmaxnm z2.s, p0/m, z2.s, z3.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
+; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT: ret
@@ -346,18 +334,15 @@ define <vscale x 4 x bfloat> @fmin_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fmin_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fmin_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fmin z2.s, p0/m, z2.s, z3.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fmin z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
+; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT: ret
@@ -416,18 +401,15 @@ define <vscale x 4 x bfloat> @fminnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale
define <vscale x 8 x bfloat> @fminnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fminnm_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fminnm z2.s, p0/m, z2.s, z3.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fminnm z3.s, p0/m, z3.s, z4.s
; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
+; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
; NOB16B16-NEXT: ret
@@ -486,18 +468,17 @@ define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) {
; NOB16B16-LABEL: fmla_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z3.s, z2.h
-; NOB16B16-NEXT: uunpklo z2.s, z2.h
+; NOB16B16-NEXT: movi v3.2d, #0000000000000000
; NOB16B16-NEXT: uunpkhi z4.s, z1.h
; NOB16B16-NEXT: uunpkhi z5.s, z0.h
; NOB16B16-NEXT: uunpklo z1.s, z1.h
; NOB16B16-NEXT: uunpklo z0.s, z0.h
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: bfmlalb z3.s, z5.h, z4.h
+; NOB16B16-NEXT: zip2 z6.h, z3.h, z2.h
+; NOB16B16-NEXT: zip1 z2.h, z3.h, z2.h
+; NOB16B16-NEXT: bfmlalb z6.s, z5.h, z4.h
; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h
-; NOB16B16-NEXT: bfcvt z0.h, p0/m, z3.s
+; NOB16B16-NEXT: bfcvt z0.h, p0/m, z6.s
; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT: uzp1 z0.h, z1.h, z0.h
; NOB16B16-NEXT: ret
@@ -676,14 +657,13 @@ define <vscale x 4 x bfloat> @fsqrt_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @fsqrt_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fsqrt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fsqrt z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fsqrt z2.s, p0/m, z2.s
; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -736,16 +716,13 @@ define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x bfloat> @fsub_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; NOB16B16-LABEL: fsub_nxv8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fsub z2.s, z3.s, z2.s
+; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h
+; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h
+; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h
+; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h
+; NOB16B16-NEXT: fsub z2.s, z4.s, z3.s
; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s
; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll b/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
index 2e3d1cc67ee51..9a45f67aad946 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
@@ -7,18 +7,17 @@ target triple = "aarch64-unknown-linux-gnu"
define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {
; SVE-LABEL: fmla_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
; SVE-NEXT: uunpkhi z4.s, z2.h
; SVE-NEXT: uunpkhi z5.s, z1.h
; SVE-NEXT: uunpklo z2.s, z2.h
; SVE-NEXT: uunpklo z1.s, z1.h
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
-; SVE-NEXT: bfmlalb z3.s, z5.h, z4.h
+; SVE-NEXT: zip2 z6.h, z3.h, z0.h
+; SVE-NEXT: zip1 z0.h, z3.h, z0.h
+; SVE-NEXT: bfmlalb z6.s, z5.h, z4.h
; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h
-; SVE-NEXT: bfcvt z1.h, p0/m, z3.s
+; SVE-NEXT: bfcvt z1.h, p0/m, z6.s
; SVE-NEXT: bfcvt z0.h, p0/m, z0.s
; SVE-NEXT: uzp1 z0.h, z0.h, z1.h
; SVE-NEXT: ret
@@ -77,19 +76,18 @@ define <vscale x 8 x bfloat> @fmls_nxv8bf16(<vscale x 8 x bfloat> %acc, <vscale
; SVE-LABEL: fmls_nxv8bf16:
; SVE: // %bb.0:
; SVE-NEXT: ptrue p0.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
; SVE-NEXT: uunpkhi z5.s, z2.h
; SVE-NEXT: uunpklo z2.s, z2.h
; SVE-NEXT: fneg z1.h, p0/m, z1.h
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
+; SVE-NEXT: zip2 z6.h, z3.h, z0.h
+; SVE-NEXT: zip1 z0.h, z3.h, z0.h
; SVE-NEXT: uunpkhi z4.s, z1.h
; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: bfmlalb z3.s, z4.h, z5.h
+; SVE-NEXT: bfmlalb z6.s, z4.h, z5.h
; SVE-NEXT: bfmlalb z0.s, z1.h, z2.h
-; SVE-NEXT: bfcvt z1.h, p0/m, z3.s
+; SVE-NEXT: bfcvt z1.h, p0/m, z6.s
; SVE-NEXT: bfcvt z0.h, p0/m, z0.s
; SVE-NEXT: uzp1 z0.h, z0.h, z1.h
; SVE-NEXT: ret
@@ -149,19 +147,18 @@ define <vscale x 2 x bfloat> @fmls_nxv2bf16(<vscale x 2 x bfloat> %acc, <vscale
define <vscale x 8 x bfloat> @fmla_sel_nxv8bf16(<vscale x 8 x i1> %pred, <vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %m1, <vscale x 8 x bfloat> %m2) {
; SVE-LABEL: fmla_sel_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: uunpkhi z5.s, z2.h
-; SVE-NEXT: uunpkhi z6.s, z1.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
+; SVE-NEXT: uunpkhi z4.s, z2.h
+; SVE-NEXT: uunpkhi z5.s, z1.h
; SVE-NEXT: uunpklo z2.s, z2.h
; SVE-NEXT: uunpklo z1.s, z1.h
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: bfmlalb z3.s, z6.h, z5.h
-; SVE-NEXT: bfmlalb z4.s, z1.h, z2.h
-; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z4.s
+; SVE-NEXT: zip2 z6.h, z3.h, z0.h
+; SVE-NEXT: zip1 z3.h, z3.h, z0.h
+; SVE-NEXT: bfmlalb z6.s, z5.h, z4.h
+; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z6.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: uzp1 z1.h, z2.h, z1.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
@@ -219,20 +216,19 @@ define <vscale x 8 x bfloat> @fmls_sel_nxv8bf16(<vscale x 8 x i1> %pred, <vscale
; SVE-LABEL: fmls_sel_nxv8bf16:
; SVE: // %bb.0:
; SVE-NEXT: ptrue p1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: uunpkhi z6.s, z2.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
+; SVE-NEXT: uunpkhi z5.s, z2.h
; SVE-NEXT: uunpklo z2.s, z2.h
; SVE-NEXT: fneg z1.h, p1/m, z1.h
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: uunpkhi z5.s, z1.h
+; SVE-NEXT: zip2 z6.h, z3.h, z0.h
+; SVE-NEXT: zip1 z3.h, z3.h, z0.h
+; SVE-NEXT: uunpkhi z4.s, z1.h
; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: bfmlalb z3.s, z5.h, z6.h
-; SVE-NEXT: bfmlalb z4.s, z1.h, z2.h
-; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z4.s
+; SVE-NEXT: bfmlalb z6.s, z4.h, z5.h
+; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z6.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: uzp1 z1.h, z2.h, z1.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
@@ -292,18 +288,15 @@ define <vscale x 2 x bfloat> @fmls_sel_nxv2bf16(<vscale x 2 x i1> %pred, <vscale
define <vscale x 8 x bfloat> @fadd_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fadd_sel_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fadd z3.s, z4.s, z3.s
+; SVE-NEXT: fadd z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -321,18 +314,15 @@ define <vscale x 8 x bfloat> @fadd_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscal
define <vscale x 8 x bfloat> @fsub_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fsub_sel_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fsub z3.s, z4.s, z3.s
+; SVE-NEXT: fsub z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -350,18 +340,15 @@ define <vscale x 8 x bfloat> @fsub_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscal
define <vscale x 8 x bfloat> @fadd_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fadd_sel_negzero_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fadd z3.s, z4.s, z3.s
+; SVE-NEXT: fadd z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -380,18 +367,15 @@ define <vscale x 8 x bfloat> @fadd_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a
define <vscale x 8 x bfloat> @fsub_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fsub_sel_negzero_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fsub z3.s, z4.s, z3.s
+; SVE-NEXT: fsub z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -417,17 +401,13 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip1 z0.h, z2.h, z0.h
; SVE-NEXT: sel z1.h, p0, z1.h, z2.h
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: fadd z2.s, z4.s, z3.s
; SVE-NEXT: fadd z0.s, z0.s, z1.s
; SVE-NEXT: bfcvt z1.h, p1/m, z2.s
; SVE-NEXT: bfcvt z0.h, p1/m, z0.s
@@ -455,19 +435,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fsub z3.s, z4.s, z3.s
+; SVE-NEXT: fsub z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -491,19 +468,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fadd z3.s, z4.s, z3.s
+; SVE-NEXT: fadd z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -527,19 +501,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fsub z3.s, z4.s, z3.s
+; SVE-NEXT: fsub z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -563,19 +534,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloa
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fadd z3.s, z4.s, z3.s
+; SVE-NEXT: fadd z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -602,17 +570,14 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloa
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
; SVE-NEXT: dupm z2.h, #0x8000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
+; SVE-NEXT: zip2 z4.h, z3.h, z0.h
+; SVE-NEXT: zip1 z0.h, z3.h, z0.h
; SVE-NEXT: sel z1.h, p0, z1.h, z2.h
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
+; SVE-NEXT: zip2 z2.h, z3.h, z1.h
+; SVE-NEXT: zip1 z1.h, z3.h, z1.h
+; SVE-NEXT: fsub z2.s, z4.s, z2.s
; SVE-NEXT: fsub z0.s, z0.s, z1.s
; SVE-NEXT: bfcvt z1.h, p1/m, z2.s
; SVE-NEXT: bfcvt z0.h, p1/m, z0.s
@@ -641,19 +606,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x b
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fadd z3.s, z4.s, z3.s
+; SVE-NEXT: fadd z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
@@ -678,19 +640,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x b
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
+; SVE-NEXT: zip2 z4.h, z2.h, z0.h
+; SVE-NEXT: zip2 z3.h, z2.h, z1.h
+; SVE-NEXT: zip1 z1.h, z2.h, z1.h
+; SVE-NEXT: zip1 z2.h, z2.h, z0.h
+; SVE-NEXT: fsub z3.s, z4.s, z3.s
+; SVE-NEXT: fsub z1.s, z2.s, z1.s
+; SVE-NEXT: bfcvt z2.h, p1/m, z3.s
; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
; SVE-NEXT: mov z0.h, p0/m, z1.h
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
index e0bf755c851b7..f9bb2997df381 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
@@ -35,16 +35,13 @@ define <vscale x 4 x i1> @fcmp_oeq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_oeq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_oeq_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmeq p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -83,16 +80,13 @@ define <vscale x 4 x i1> @fcmp_ogt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ogt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ogt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -131,16 +125,13 @@ define <vscale x 4 x i1> @fcmp_oge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_oge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_oge_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -179,16 +170,13 @@ define <vscale x 4 x i1> @fcmp_olt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_olt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_olt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -227,16 +215,13 @@ define <vscale x 4 x i1> @fcmp_ole_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ole_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ole_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -279,17 +264,14 @@ define <vscale x 4 x i1> @fcmp_one_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_one_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_one_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
-; CHECK-NEXT: fcmgt p2.s, p0/z, z2.s, z3.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
+; CHECK-NEXT: fcmgt p2.s, p0/z, z3.s, z4.s
; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z0.s
; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: mov p1.b, p2/m, p2.b
@@ -333,16 +315,13 @@ define <vscale x 4 x i1> @fcmp_ord_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ord_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ord_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmuo p2.s, p0/z, z0.s, z1.s
; CHECK-NEXT: not p1.b, p0/z, p1.b
; CHECK-NEXT: not p0.b, p0/z, p2.b
@@ -387,17 +366,14 @@ define <vscale x 4 x i1> @fcmp_ueq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ueq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ueq_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
-; CHECK-NEXT: fcmeq p2.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s
+; CHECK-NEXT: fcmeq p2.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmuo p3.s, p0/z, z0.s, z1.s
; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: mov p1.b, p2/m, p2.b
@@ -441,16 +417,13 @@ define <vscale x 4 x i1> @fcmp_ugt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ugt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ugt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z0.s
; CHECK-NEXT: not p1.b, p0/z, p1.b
; CHECK-NEXT: not p0.b, p0/z, p2.b
@@ -493,16 +466,13 @@ define <vscale x 4 x i1> @fcmp_uge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_uge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_uge_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p2.s, p0/z, z1.s, z0.s
; CHECK-NEXT: not p1.b, p0/z, p1.b
; CHECK-NEXT: not p0.b, p0/z, p2.b
@@ -545,16 +515,13 @@ define <vscale x 4 x i1> @fcmp_ult_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ult_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ult_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z1.s
; CHECK-NEXT: not p1.b, p0/z, p1.b
; CHECK-NEXT: not p0.b, p0/z, p2.b
@@ -597,16 +564,13 @@ define <vscale x 4 x i1> @fcmp_ule_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_ule_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ule_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p2.s, p0/z, z0.s, z1.s
; CHECK-NEXT: not p1.b, p0/z, p1.b
; CHECK-NEXT: not p0.b, p0/z, p2.b
@@ -647,16 +611,13 @@ define <vscale x 4 x i1> @fcmp_une_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_une_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_une_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmne p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -695,16 +656,13 @@ define <vscale x 4 x i1> @fcmp_uno_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
define <vscale x 8 x i1> @fcmp_uno_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_uno_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -743,16 +701,13 @@ define <vscale x 4 x i1> @fcmp_eq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_eq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_eq_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmeq p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -791,16 +746,13 @@ define <vscale x 4 x i1> @fcmp_gt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_gt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_gt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -839,16 +791,13 @@ define <vscale x 4 x i1> @fcmp_ge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_ge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ge_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -887,16 +836,13 @@ define <vscale x 4 x i1> @fcmp_lt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_lt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_lt_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -935,16 +881,13 @@ define <vscale x 4 x i1> @fcmp_le_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_le_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_le_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: uunpkhi z3.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z0.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
@@ -983,16 +926,13 @@ define <vscale x 4 x i1> @fcmp_ne_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4
define <vscale x 8 x i1> @fcmp_ne_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
; CHECK-LABEL: fcmp_ne_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z2.s, z1.h
-; CHECK-NEXT: uunpkhi z3.s, z0.h
-; CHECK-NEXT: uunpklo z1.s, z1.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z2.s, z2.s, #16
-; CHECK-NEXT: lsl z3.s, z3.s, #16
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: zip2 z3.h, z2.h, z1.h
+; CHECK-NEXT: zip2 z4.h, z2.h, z0.h
+; CHECK-NEXT: zip1 z1.h, z2.h, z1.h
+; CHECK-NEXT: zip1 z0.h, z2.h, z0.h
+; CHECK-NEXT: fcmne p1.s, p0/z, z4.s, z3.s
; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
index 120ab7cc4552e..a539497c80d58 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
@@ -31,10 +31,10 @@ define <vscale x 4 x float> @fpext_nxv4bf16_to_nxv4f32(<vscale x 4 x bfloat> %a)
define <vscale x 8 x float> @fpext_nxv8bf16_to_nxv8f32(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fpext_nxv8bf16_to_nxv8f32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpklo z1.s, z0.h
-; CHECK-NEXT: uunpkhi z2.s, z0.h
-; CHECK-NEXT: lsl z0.s, z1.s, #16
-; CHECK-NEXT: lsl z1.s, z2.s, #16
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: zip1 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip2 z1.h, z1.h, z0.h
+; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ret
%res = fpext <vscale x 8 x bfloat> %a to <vscale x 8 x float>
ret <vscale x 8 x float> %res
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
index bdfe90c8a6bb7..3e7f8424c1334 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll
@@ -125,15 +125,14 @@ define <vscale x 4 x i64> @fptosi_nxv4bf16_to_nxv4i64(<vscale x 4 x bfloat> %a)
define <vscale x 8 x i1> @fptosi_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
; CHECK-NEXT: ret
%res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i1>
@@ -143,14 +142,13 @@ define <vscale x 8 x i1> @fptosi_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) {
define <vscale x 8 x i8> @fptosi_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: ret
%res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i8>
ret <vscale x 8 x i8> %res
@@ -159,14 +157,13 @@ define <vscale x 8 x i8> @fptosi_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) {
define <vscale x 8 x i16> @fptosi_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: ret
%res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i16>
ret <vscale x 8 x i16> %res
@@ -175,15 +172,13 @@ define <vscale x 8 x i16> @fptosi_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a)
define <vscale x 8 x i32> @fptosi_nxv8bf16_to_nxv8i32(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpklo z1.s, z0.h
-; CHECK-NEXT: uunpkhi z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z2.s, z0.s, #16
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: fcvtzs z0.s, p0/m, z1.s
-; CHECK-NEXT: movprfx z1, z2
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z2.s
+; CHECK-NEXT: zip1 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip2 z1.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: movprfx z0, z2
+; CHECK-NEXT: fcvtzs z0.s, p0/m, z2.s
; CHECK-NEXT: ret
%res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i32>
ret <vscale x 8 x i32> %res
@@ -338,15 +333,14 @@ define <vscale x 4 x i64> @fptoui_nxv4bf16_to_nxv4i64(<vscale x 4 x bfloat> %a)
define <vscale x 8 x i1> @fptoui_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0
; CHECK-NEXT: ret
%res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i1>
@@ -356,14 +350,13 @@ define <vscale x 8 x i1> @fptoui_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) {
define <vscale x 8 x i8> @fptoui_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: ret
%res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i8>
ret <vscale x 8 x i8> %res
@@ -372,14 +365,13 @@ define <vscale x 8 x i8> @fptoui_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) {
define <vscale x 8 x i16> @fptoui_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s
; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s
-; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
+; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h
; CHECK-NEXT: ret
%res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i16>
ret <vscale x 8 x i16> %res
@@ -388,15 +380,13 @@ define <vscale x 8 x i16> @fptoui_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a)
define <vscale x 8 x i32> @fptoui_nxv8bf16_to_nxv8i32(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpklo z1.s, z0.h
-; CHECK-NEXT: uunpkhi z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z2.s, z0.s, #16
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: fcvtzu z0.s, p0/m, z1.s
-; CHECK-NEXT: movprfx z1, z2
-; CHECK-NEXT: fcvtzu z1.s, p0/m, z2.s
+; CHECK-NEXT: zip1 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip2 z1.h, z1.h, z0.h
+; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s
+; CHECK-NEXT: movprfx z0, z2
+; CHECK-NEXT: fcvtzu z0.s, p0/m, z2.s
; CHECK-NEXT: ret
%res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i32>
ret <vscale x 8 x i32> %res
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll
index c1f64892ef895..a888238503e23 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll
@@ -33,12 +33,11 @@ define bfloat @faddv_nxv4bf16(<vscale x 4 x bfloat> %a) {
define bfloat @faddv_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: faddv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fadd z0.s, z0.s, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fadd z0.s, z0.s, z2.s
; CHECK-NEXT: faddv s0, p0, z0.s
; CHECK-NEXT: bfcvt h0, s0
; CHECK-NEXT: ret
@@ -75,12 +74,11 @@ define bfloat @fmaxv_nxv4bf16(<vscale x 4 x bfloat> %a) {
define bfloat @fmaxv_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fmaxv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z2.s
; CHECK-NEXT: fmaxnmv s0, p0, z0.s
; CHECK-NEXT: bfcvt h0, s0
; CHECK-NEXT: ret
@@ -117,12 +115,11 @@ define bfloat @fminv_nxv4bf16(<vscale x 4 x bfloat> %a) {
define bfloat @fminv_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fminv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z2.s
; CHECK-NEXT: fminnmv s0, p0, z0.s
; CHECK-NEXT: bfcvt h0, s0
; CHECK-NEXT: ret
@@ -159,12 +156,11 @@ define bfloat @fmaximumv_nxv4bf16(<vscale x 4 x bfloat> %a) {
define bfloat @fmaximumv_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fmaximumv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z2.s
; CHECK-NEXT: fmaxv s0, p0, z0.s
; CHECK-NEXT: bfcvt h0, s0
; CHECK-NEXT: ret
@@ -201,12 +197,11 @@ define bfloat @fminimumv_nxv4bf16(<vscale x 4 x bfloat> %a) {
define bfloat @fminimumv_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: fminimumv_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z2.s
; CHECK-NEXT: fminv s0, p0, z0.s
; CHECK-NEXT: bfcvt h0, s0
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll b/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll
index 65d273d120982..80b54ff31aaf0 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll
@@ -35,14 +35,13 @@ define <vscale x 4 x bfloat> @frintp_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frintp_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintp_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frintp z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frintp z2.s, p0/m, z2.s
; CHECK-NEXT: frintp z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -81,14 +80,13 @@ define <vscale x 4 x bfloat> @frintm_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frintm_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintm_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frintm z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frintm z2.s, p0/m, z2.s
; CHECK-NEXT: frintm z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -127,14 +125,13 @@ define <vscale x 4 x bfloat> @frinti_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frinti_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frinti_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frinti z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frinti z2.s, p0/m, z2.s
; CHECK-NEXT: frinti z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -173,14 +170,13 @@ define <vscale x 4 x bfloat> @frintx_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frintx_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintx_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frintx z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frintx z2.s, p0/m, z2.s
; CHECK-NEXT: frintx z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -219,14 +215,13 @@ define <vscale x 4 x bfloat> @frinta_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frinta_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frinta_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frinta z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frinta z2.s, p0/m, z2.s
; CHECK-NEXT: frinta z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -265,14 +260,13 @@ define <vscale x 4 x bfloat> @frintn_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frintn_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintn_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frintn z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frintn z2.s, p0/m, z2.s
; CHECK-NEXT: frintn z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
@@ -311,14 +305,13 @@ define <vscale x 4 x bfloat> @frintz_nxv4bf16(<vscale x 4 x bfloat> %a) {
define <vscale x 8 x bfloat> @frintz_nxv8bf16(<vscale x 8 x bfloat> %a) {
; CHECK-LABEL: frintz_nxv8bf16:
; CHECK: // %bb.0:
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: movi v1.2d, #0000000000000000
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: lsl z1.s, z1.s, #16
-; CHECK-NEXT: lsl z0.s, z0.s, #16
-; CHECK-NEXT: frintz z1.s, p0/m, z1.s
+; CHECK-NEXT: zip2 z2.h, z1.h, z0.h
+; CHECK-NEXT: zip1 z0.h, z1.h, z0.h
+; CHECK-NEXT: frintz z2.s, p0/m, z2.s
; CHECK-NEXT: frintz z0.s, p0/m, z0.s
-; CHECK-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s
; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s
; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h
; CHECK-NEXT: ret
More information about the llvm-commits
mailing list