[llvm] [AArch64] Lower fixed width bf16 fpround (PR #209411)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 02:24:38 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/209411
Similar to #209194, this was previously hitting an error with an illegal FP_ROUND_MERGE_PASSTHRU. This time we lower the fptrunc to a scalable fptrunc and let it legalize naturally. On SVE systems a f64->bf16 fptrunc will fail to legalize, but lowers successfully with sve2 with fcvtn. A store(fptrunc) -> truncstore combine was disabled too as it fails with bf16.
>From 296996ca8d883e01b9108b9a7cdfba23fb6ab722 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Tue, 14 Jul 2026 10:12:51 +0100
Subject: [PATCH] [AArch64] Lower fixed width bf16 fpround
Simlar to #209206, this was previously hitting an error with an illegal
FP_ROUND_MERGE_PASSTHRU. This time we lower the fptrunc to a scalable fptrunc
and let it legalize naturally. On SVE systems a f64->bf16 fptrunc will fail to
legalize, but lowers sucessfully with sve2 with fcvtn. A store(fptrunc) ->
truncstore combine was disabled too as it fails with bf16.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 9 +-
.../sve-fixed-length-fp-extend-trunc-bf16.ll | 1868 ++++++++++++++++-
2 files changed, 1866 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0a8a116098275..ab19d8e905aa1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27362,11 +27362,12 @@ static SDValue performSTORECombine(SDNode *N,
// We purposefully don't care about legality of the nodes here as we know
// they can be split down into something legal.
if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND &&
- Value.getNode()->hasOneUse() && ST->isUnindexed() &&
+ Value->hasOneUse() && ST->isUnindexed() &&
Subtarget->useSVEForFixedLengthVectors() &&
ValueVT.isFixedLengthVector() &&
ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() &&
- hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()))
+ hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()) &&
+ Value.getValueType().getScalarType() != MVT::bf16)
return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT,
ST->getMemOperand());
@@ -33985,11 +33986,9 @@ AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
EVT ContainerSrcVT = getContainerForFixedLengthVector(DAG, SrcVT);
EVT RoundVT = ContainerSrcVT.changeVectorElementType(
*DAG.getContext(), VT.getVectorElementType());
- SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
- Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val,
- Op.getOperand(1), DAG.getPOISON(RoundVT));
+ Val = DAG.getNode(Op.getOpcode(), DL, RoundVT, Val, Op.getOperand(1));
Val = getSVESafeBitCast(ContainerSrcVT.changeTypeToInteger(), Val, DAG);
Val = convertFromScalableVector(DAG, SrcVT.changeTypeToInteger(), Val);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
index 98d6b7681bbf6..0ab8786387d31 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_256_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_512_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048,VBITS_GE_2048_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_256_BF16
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_512_BF16
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048,VBITS_GE_2048_BF16
define void @fcvt_v2bf16_v2f32(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v2bf16_v2f32:
@@ -992,3 +992,1859 @@ define void @fcvt_v32bf16_v32f64(ptr %a, ptr %b) vscale_range(16,0) {
store <32 x double> %res, ptr %b
ret void
}
+
+
+define void @fcvt_v2f32_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_256_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_256_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_256_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_256_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_256_BASE-NEXT: str s0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_512_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_512_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_512_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_512_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_512_BASE-NEXT: str s0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_2048_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_2048_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_2048_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_2048_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_2048_BASE-NEXT: str s0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_256_BF16-NEXT: str s0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_512_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_512_BF16-NEXT: str s0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_2048_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_2048_BF16-NEXT: str s0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <2 x float>, ptr %a
+ %res = fptrunc <2 x float> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f32_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_256_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_256_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_256_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_256_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_256_BASE-NEXT: str d0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_512_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_512_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_512_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_512_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_512_BASE-NEXT: str d0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_2048_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_2048_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_2048_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_2048_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_2048_BASE-NEXT: str d0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_256_BF16-NEXT: str d0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_512_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_512_BF16-NEXT: str d0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_2048_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_2048_BF16-NEXT: str d0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <4 x float>, ptr %a
+ %res = fptrunc <4 x float> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f32_v8bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: str q0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: str q0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: str q0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: str q0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: str q0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: str q0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <8 x float>, ptr %a
+ %res = fptrunc <8 x float> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f32_v16bf16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BASE-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256_BASE-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: add z4.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_256_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: add z3.s, z3.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z1.s, z5.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_256_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: stp q1, q0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_512_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z5.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: lsr z4.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_512_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_512_BASE-NEXT: add z1.s, z4.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_512_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl16
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z5.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: lsr z4.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z4.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BF16-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256_BF16-NEXT: ptrue p1.s
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256_BF16-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p1/m, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p1/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: stp q1, q0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl16
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #32
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <16 x float>, ptr %a
+ %res = fptrunc <16 x float> %op1 to <16 x bfloat>
+ store <16 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v32f32_v32bf16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl32
+; VBITS_GE_256_BASE-NEXT: mov z3.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_256_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_256_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_256_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_256_BASE-NEXT: lsr z4.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z16.s, z0.s, z3.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: movprfx z7, z1
+; VBITS_GE_256_BASE-NEXT: ext z7.b, z7.b, z1.b, #32
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z6.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: add z17.s, z2.s, z3.s
+; VBITS_GE_256_BASE-NEXT: add z18.s, z1.s, z3.s
+; VBITS_GE_256_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_256_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: lsr z19.s, z7.s, #16
+; VBITS_GE_256_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z3.s, z7.s, z3.s
+; VBITS_GE_256_BASE-NEXT: add z4.s, z4.s, z16.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z5.s, z5.s, z17.s
+; VBITS_GE_256_BASE-NEXT: add z6.s, z6.s, z18.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s
+; VBITS_GE_256_BASE-NEXT: orr z7.s, z7.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z3.s, z19.s, z3.s
+; VBITS_GE_256_BASE-NEXT: sel z2.s, p2, z2.s, z5.s
+; VBITS_GE_256_BASE-NEXT: sel z1.s, p3, z1.s, z6.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: mov z3.s, p0/m, z7.s
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_256_BASE-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl32
+; VBITS_GE_512_BASE-NEXT: mov z3.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_512_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_512_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_512_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_512_BASE-NEXT: lsr z4.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z16.s, z0.s, z3.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: movprfx z7, z1
+; VBITS_GE_512_BASE-NEXT: ext z7.b, z7.b, z1.b, #32
+; VBITS_GE_512_BASE-NEXT: lsr z5.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z6.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: add z17.s, z2.s, z3.s
+; VBITS_GE_512_BASE-NEXT: add z18.s, z1.s, z3.s
+; VBITS_GE_512_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_512_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_512_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: lsr z19.s, z7.s, #16
+; VBITS_GE_512_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_512_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z3.s, z7.s, z3.s
+; VBITS_GE_512_BASE-NEXT: add z4.s, z4.s, z16.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z5.s, z5.s, z17.s
+; VBITS_GE_512_BASE-NEXT: add z6.s, z6.s, z18.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s
+; VBITS_GE_512_BASE-NEXT: orr z7.s, z7.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p1, z0.s, z4.s
+; VBITS_GE_512_BASE-NEXT: add z3.s, z19.s, z3.s
+; VBITS_GE_512_BASE-NEXT: sel z2.s, p2, z2.s, z5.s
+; VBITS_GE_512_BASE-NEXT: sel z1.s, p3, z1.s, z6.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: mov z3.s, p0/m, z7.s
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_512_BASE-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl32
+; VBITS_GE_2048_BASE-NEXT: mov z3.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_2048_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_2048_BASE-NEXT: lsr z4.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z16.s, z0.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: movprfx z7, z1
+; VBITS_GE_2048_BASE-NEXT: ext z7.b, z7.b, z1.b, #32
+; VBITS_GE_2048_BASE-NEXT: lsr z5.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z6.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z17.s, z2.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: add z18.s, z1.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: lsr z19.s, z7.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z3.s, z7.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: add z4.s, z4.s, z16.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z5.s, z5.s, z17.s
+; VBITS_GE_2048_BASE-NEXT: add z6.s, z6.s, z18.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s
+; VBITS_GE_2048_BASE-NEXT: orr z7.s, z7.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p1, z0.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: add z3.s, z19.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: sel z2.s, p2, z2.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: sel z1.s, p3, z1.s, z6.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: mov z3.s, p0/m, z7.s
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_2048_BASE-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl32
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s
+; VBITS_GE_256_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_256_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_256_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_256_BF16-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_256_BF16-NEXT: ext z3.b, z3.b, z1.b, #32
+; VBITS_GE_256_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_256_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl32
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_512_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_512_BF16-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_512_BF16-NEXT: ext z3.b, z3.b, z1.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_512_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v32f32_v32bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl32
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_2048_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BF16-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_2048_BF16-NEXT: ext z3.b, z3.b, z1.b, #32
+; VBITS_GE_2048_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_2048_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <32 x float>, ptr %a
+ %res = fptrunc <32 x float> %op1 to <32 x bfloat>
+ store <32 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v64f32_v64bf16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl64
+; VBITS_GE_256_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_256_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_256_BASE-NEXT: movprfx z3, z0
+; VBITS_GE_256_BASE-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_256_BASE-NEXT: movprfx z4, z0
+; VBITS_GE_256_BASE-NEXT: ext z4.b, z4.b, z0.b, #64
+; VBITS_GE_256_BASE-NEXT: lsr z16.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: lsr z22.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: movprfx z5, z1
+; VBITS_GE_256_BASE-NEXT: ext z5.b, z5.b, z1.b, #32
+; VBITS_GE_256_BASE-NEXT: lsr z6.s, z3.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z7.s, z4.s, #16
+; VBITS_GE_256_BASE-NEXT: movprfx z17, z1
+; VBITS_GE_256_BASE-NEXT: ext z17.b, z17.b, z1.b, #64
+; VBITS_GE_256_BASE-NEXT: add z23.s, z1.s, z2.s
+; VBITS_GE_256_BASE-NEXT: movprfx z18, z4
+; VBITS_GE_256_BASE-NEXT: ext z18.b, z18.b, z4.b, #32
+; VBITS_GE_256_BASE-NEXT: add z19.s, z3.s, z2.s
+; VBITS_GE_256_BASE-NEXT: add z21.s, z4.s, z2.s
+; VBITS_GE_256_BASE-NEXT: and z22.s, z22.s, #0x1
+; VBITS_GE_256_BASE-NEXT: lsr z20.s, z5.s, #16
+; VBITS_GE_256_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_256_BASE-NEXT: movprfx z24, z17
+; VBITS_GE_256_BASE-NEXT: ext z24.b, z24.b, z17.b, #32
+; VBITS_GE_256_BASE-NEXT: and z16.s, z16.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z26.s, z5.s, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z25.s, z18.s, #16
+; VBITS_GE_256_BASE-NEXT: add z22.s, z22.s, z23.s
+; VBITS_GE_256_BASE-NEXT: add z23.s, z0.s, z2.s
+; VBITS_GE_256_BASE-NEXT: and z20.s, z20.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s
+; VBITS_GE_256_BASE-NEXT: add z6.s, z6.s, z19.s
+; VBITS_GE_256_BASE-NEXT: add z7.s, z7.s, z21.s
+; VBITS_GE_256_BASE-NEXT: lsr z21.s, z17.s, #16
+; VBITS_GE_256_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: add z16.s, z16.s, z23.s
+; VBITS_GE_256_BASE-NEXT: lsr z23.s, z24.s, #16
+; VBITS_GE_256_BASE-NEXT: add z20.s, z20.s, z26.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s
+; VBITS_GE_256_BASE-NEXT: orr z5.s, z5.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: add z19.s, z18.s, z2.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z26.s, z17.s, z2.s
+; VBITS_GE_256_BASE-NEXT: and z25.s, z25.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: and z21.s, z21.s, #0x1
+; VBITS_GE_256_BASE-NEXT: orr z4.s, z4.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: sel z3.s, p2, z3.s, z6.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p5, z0.s, z16.s
+; VBITS_GE_256_BASE-NEXT: and z23.s, z23.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z2.s, z24.s, z2.s
+; VBITS_GE_256_BASE-NEXT: sel z5.s, p1, z5.s, z20.s
+; VBITS_GE_256_BASE-NEXT: sel z4.s, p3, z4.s, z7.s
+; VBITS_GE_256_BASE-NEXT: add z6.s, z25.s, z19.s
+; VBITS_GE_256_BASE-NEXT: add z7.s, z21.s, z26.s
+; VBITS_GE_256_BASE-NEXT: sel z1.s, p4, z1.s, z22.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_256_BASE-NEXT: add z2.s, z23.s, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z5.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z4.s, z4.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z5.h, z5.h, z5.h
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z18.s, z18.s
+; VBITS_GE_256_BASE-NEXT: orr z18.s, z18.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: stp q0, q3, [x1]
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: fcmuo p2.s, p0/z, z17.s, z17.s
+; VBITS_GE_256_BASE-NEXT: orr z17.s, z17.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z24.s, z24.s
+; VBITS_GE_256_BASE-NEXT: orr z24.s, z24.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: stp q1, q5, [x1, #64]
+; VBITS_GE_256_BASE-NEXT: mov z6.s, p1/m, z18.s
+; VBITS_GE_256_BASE-NEXT: mov z7.s, p2/m, z17.s
+; VBITS_GE_256_BASE-NEXT: mov z2.s, p0/m, z24.s
+; VBITS_GE_256_BASE-NEXT: lsr z6.s, z6.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z7.s, z7.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z2.h, z4.h, z4.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z3.h, z6.h, z6.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z7.h, z7.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: stp q2, q3, [x1, #32]
+; VBITS_GE_256_BASE-NEXT: stp q1, q0, [x1, #96]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl64
+; VBITS_GE_512_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_512_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_512_BASE-NEXT: movprfx z3, z0
+; VBITS_GE_512_BASE-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_512_BASE-NEXT: movprfx z4, z0
+; VBITS_GE_512_BASE-NEXT: ext z4.b, z4.b, z0.b, #64
+; VBITS_GE_512_BASE-NEXT: lsr z16.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: lsr z22.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: movprfx z5, z1
+; VBITS_GE_512_BASE-NEXT: ext z5.b, z5.b, z1.b, #32
+; VBITS_GE_512_BASE-NEXT: lsr z6.s, z3.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z7.s, z4.s, #16
+; VBITS_GE_512_BASE-NEXT: movprfx z17, z1
+; VBITS_GE_512_BASE-NEXT: ext z17.b, z17.b, z1.b, #64
+; VBITS_GE_512_BASE-NEXT: add z23.s, z1.s, z2.s
+; VBITS_GE_512_BASE-NEXT: movprfx z18, z4
+; VBITS_GE_512_BASE-NEXT: ext z18.b, z18.b, z4.b, #32
+; VBITS_GE_512_BASE-NEXT: add z19.s, z3.s, z2.s
+; VBITS_GE_512_BASE-NEXT: add z21.s, z4.s, z2.s
+; VBITS_GE_512_BASE-NEXT: and z22.s, z22.s, #0x1
+; VBITS_GE_512_BASE-NEXT: lsr z20.s, z5.s, #16
+; VBITS_GE_512_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_512_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_512_BASE-NEXT: movprfx z24, z17
+; VBITS_GE_512_BASE-NEXT: ext z24.b, z24.b, z17.b, #32
+; VBITS_GE_512_BASE-NEXT: and z16.s, z16.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z26.s, z5.s, z2.s
+; VBITS_GE_512_BASE-NEXT: lsr z25.s, z18.s, #16
+; VBITS_GE_512_BASE-NEXT: add z22.s, z22.s, z23.s
+; VBITS_GE_512_BASE-NEXT: add z23.s, z0.s, z2.s
+; VBITS_GE_512_BASE-NEXT: and z20.s, z20.s, #0x1
+; VBITS_GE_512_BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s
+; VBITS_GE_512_BASE-NEXT: add z6.s, z6.s, z19.s
+; VBITS_GE_512_BASE-NEXT: add z7.s, z7.s, z21.s
+; VBITS_GE_512_BASE-NEXT: lsr z21.s, z17.s, #16
+; VBITS_GE_512_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: add z16.s, z16.s, z23.s
+; VBITS_GE_512_BASE-NEXT: lsr z23.s, z24.s, #16
+; VBITS_GE_512_BASE-NEXT: add z20.s, z20.s, z26.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s
+; VBITS_GE_512_BASE-NEXT: orr z5.s, z5.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: add z19.s, z18.s, z2.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s
+; VBITS_GE_512_BASE-NEXT: add z26.s, z17.s, z2.s
+; VBITS_GE_512_BASE-NEXT: and z25.s, z25.s, #0x1
+; VBITS_GE_512_BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: and z21.s, z21.s, #0x1
+; VBITS_GE_512_BASE-NEXT: orr z4.s, z4.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: sel z3.s, p2, z3.s, z6.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p5, z0.s, z16.s
+; VBITS_GE_512_BASE-NEXT: and z23.s, z23.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z2.s, z24.s, z2.s
+; VBITS_GE_512_BASE-NEXT: sel z5.s, p1, z5.s, z20.s
+; VBITS_GE_512_BASE-NEXT: sel z4.s, p3, z4.s, z7.s
+; VBITS_GE_512_BASE-NEXT: add z6.s, z25.s, z19.s
+; VBITS_GE_512_BASE-NEXT: add z7.s, z21.s, z26.s
+; VBITS_GE_512_BASE-NEXT: sel z1.s, p4, z1.s, z22.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_512_BASE-NEXT: add z2.s, z23.s, z2.s
+; VBITS_GE_512_BASE-NEXT: lsr z5.s, z5.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z4.s, z4.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z5.h, z5.h, z5.h
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z18.s, z18.s
+; VBITS_GE_512_BASE-NEXT: orr z18.s, z18.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: stp q0, q3, [x1]
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: fcmuo p2.s, p0/z, z17.s, z17.s
+; VBITS_GE_512_BASE-NEXT: orr z17.s, z17.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z24.s, z24.s
+; VBITS_GE_512_BASE-NEXT: orr z24.s, z24.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: stp q1, q5, [x1, #64]
+; VBITS_GE_512_BASE-NEXT: mov z6.s, p1/m, z18.s
+; VBITS_GE_512_BASE-NEXT: mov z7.s, p2/m, z17.s
+; VBITS_GE_512_BASE-NEXT: mov z2.s, p0/m, z24.s
+; VBITS_GE_512_BASE-NEXT: lsr z6.s, z6.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z7.s, z7.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z2.h, z4.h, z4.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z3.h, z6.h, z6.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z7.h, z7.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: stp q2, q3, [x1, #32]
+; VBITS_GE_512_BASE-NEXT: stp q1, q0, [x1, #96]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl64
+; VBITS_GE_2048_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_2048_BASE-NEXT: movprfx z3, z0
+; VBITS_GE_2048_BASE-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_2048_BASE-NEXT: movprfx z4, z0
+; VBITS_GE_2048_BASE-NEXT: ext z4.b, z4.b, z0.b, #64
+; VBITS_GE_2048_BASE-NEXT: lsr z16.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: lsr z22.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: movprfx z5, z1
+; VBITS_GE_2048_BASE-NEXT: ext z5.b, z5.b, z1.b, #32
+; VBITS_GE_2048_BASE-NEXT: lsr z6.s, z3.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z7.s, z4.s, #16
+; VBITS_GE_2048_BASE-NEXT: movprfx z17, z1
+; VBITS_GE_2048_BASE-NEXT: ext z17.b, z17.b, z1.b, #64
+; VBITS_GE_2048_BASE-NEXT: add z23.s, z1.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: movprfx z18, z4
+; VBITS_GE_2048_BASE-NEXT: ext z18.b, z18.b, z4.b, #32
+; VBITS_GE_2048_BASE-NEXT: add z19.s, z3.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: add z21.s, z4.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: and z22.s, z22.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: lsr z20.s, z5.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: movprfx z24, z17
+; VBITS_GE_2048_BASE-NEXT: ext z24.b, z24.b, z17.b, #32
+; VBITS_GE_2048_BASE-NEXT: and z16.s, z16.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z26.s, z5.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: lsr z25.s, z18.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z22.s, z22.s, z23.s
+; VBITS_GE_2048_BASE-NEXT: add z23.s, z0.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: and z20.s, z20.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: add z6.s, z6.s, z19.s
+; VBITS_GE_2048_BASE-NEXT: add z7.s, z7.s, z21.s
+; VBITS_GE_2048_BASE-NEXT: lsr z21.s, z17.s, #16
+; VBITS_GE_2048_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: add z16.s, z16.s, z23.s
+; VBITS_GE_2048_BASE-NEXT: lsr z23.s, z24.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z20.s, z20.s, z26.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: orr z5.s, z5.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: add z19.s, z18.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: add z26.s, z17.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: and z25.s, z25.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: and z21.s, z21.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: orr z4.s, z4.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: sel z3.s, p2, z3.s, z6.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p5, z0.s, z16.s
+; VBITS_GE_2048_BASE-NEXT: and z23.s, z23.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z2.s, z24.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: sel z5.s, p1, z5.s, z20.s
+; VBITS_GE_2048_BASE-NEXT: sel z4.s, p3, z4.s, z7.s
+; VBITS_GE_2048_BASE-NEXT: add z6.s, z25.s, z19.s
+; VBITS_GE_2048_BASE-NEXT: add z7.s, z21.s, z26.s
+; VBITS_GE_2048_BASE-NEXT: sel z1.s, p4, z1.s, z22.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z2.s, z23.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: lsr z5.s, z5.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z4.s, z4.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z5.h, z5.h, z5.h
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z18.s, z18.s
+; VBITS_GE_2048_BASE-NEXT: orr z18.s, z18.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: stp q0, q3, [x1]
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: fcmuo p2.s, p0/z, z17.s, z17.s
+; VBITS_GE_2048_BASE-NEXT: orr z17.s, z17.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z24.s, z24.s
+; VBITS_GE_2048_BASE-NEXT: orr z24.s, z24.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: stp q1, q5, [x1, #64]
+; VBITS_GE_2048_BASE-NEXT: mov z6.s, p1/m, z18.s
+; VBITS_GE_2048_BASE-NEXT: mov z7.s, p2/m, z17.s
+; VBITS_GE_2048_BASE-NEXT: mov z2.s, p0/m, z24.s
+; VBITS_GE_2048_BASE-NEXT: lsr z6.s, z6.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z7.s, z7.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z2.h, z4.h, z4.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z3.h, z6.h, z6.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z7.h, z7.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: stp q2, q3, [x1, #32]
+; VBITS_GE_2048_BASE-NEXT: stp q1, q0, [x1, #96]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl64
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s
+; VBITS_GE_256_BF16-NEXT: movprfx z3, z0
+; VBITS_GE_256_BF16-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_256_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_256_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_256_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_256_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_256_BF16-NEXT: movprfx z4, z1
+; VBITS_GE_256_BF16-NEXT: ext z4.b, z4.b, z1.b, #32
+; VBITS_GE_256_BF16-NEXT: movprfx z5, z1
+; VBITS_GE_256_BF16-NEXT: ext z5.b, z5.b, z1.b, #64
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_256_BF16-NEXT: movprfx z6, z2
+; VBITS_GE_256_BF16-NEXT: ext z6.b, z6.b, z2.b, #32
+; VBITS_GE_256_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: bfcvt z4.h, p0/m, z4.s
+; VBITS_GE_256_BF16-NEXT: movprfx z7, z5
+; VBITS_GE_256_BF16-NEXT: ext z7.b, z7.b, z5.b, #32
+; VBITS_GE_256_BF16-NEXT: bfcvt z5.h, p0/m, z5.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BF16-NEXT: bfcvt z6.h, p0/m, z6.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256_BF16-NEXT: stp q0, q3, [x1]
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z7.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z4.h, z4.h, z4.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z3.h, z5.h, z5.h
+; VBITS_GE_256_BF16-NEXT: stp q1, q4, [x1, #64]
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z6.h, z6.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: stp q2, q1, [x1, #32]
+; VBITS_GE_256_BF16-NEXT: stp q3, q0, [x1, #96]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl64
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: movprfx z3, z0
+; VBITS_GE_512_BF16-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_512_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_512_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_512_BF16-NEXT: movprfx z4, z1
+; VBITS_GE_512_BF16-NEXT: ext z4.b, z4.b, z1.b, #32
+; VBITS_GE_512_BF16-NEXT: movprfx z5, z1
+; VBITS_GE_512_BF16-NEXT: ext z5.b, z5.b, z1.b, #64
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: movprfx z6, z2
+; VBITS_GE_512_BF16-NEXT: ext z6.b, z6.b, z2.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: bfcvt z4.h, p0/m, z4.s
+; VBITS_GE_512_BF16-NEXT: movprfx z7, z5
+; VBITS_GE_512_BF16-NEXT: ext z7.b, z7.b, z5.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z5.h, p0/m, z5.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BF16-NEXT: bfcvt z6.h, p0/m, z6.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q3, [x1]
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z7.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z4.h, z4.h, z4.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z3.h, z5.h, z5.h
+; VBITS_GE_512_BF16-NEXT: stp q1, q4, [x1, #64]
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z6.h, z6.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: stp q2, q1, [x1, #32]
+; VBITS_GE_512_BF16-NEXT: stp q3, q0, [x1, #96]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v64f32_v64bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl64
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z3, z0
+; VBITS_GE_2048_BF16-NEXT: ext z3.b, z3.b, z0.b, #32
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_2048_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z4, z1
+; VBITS_GE_2048_BF16-NEXT: ext z4.b, z4.b, z1.b, #32
+; VBITS_GE_2048_BF16-NEXT: movprfx z5, z1
+; VBITS_GE_2048_BF16-NEXT: ext z5.b, z5.b, z1.b, #64
+; VBITS_GE_2048_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z6, z2
+; VBITS_GE_2048_BF16-NEXT: ext z6.b, z6.b, z2.b, #32
+; VBITS_GE_2048_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: bfcvt z4.h, p0/m, z4.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z7, z5
+; VBITS_GE_2048_BF16-NEXT: ext z7.b, z7.b, z5.b, #32
+; VBITS_GE_2048_BF16-NEXT: bfcvt z5.h, p0/m, z5.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BF16-NEXT: bfcvt z6.h, p0/m, z6.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_2048_BF16-NEXT: stp q0, q3, [x1]
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z7.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z4.h, z4.h, z4.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z3.h, z5.h, z5.h
+; VBITS_GE_2048_BF16-NEXT: stp q1, q4, [x1, #64]
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z6.h, z6.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: stp q2, q1, [x1, #32]
+; VBITS_GE_2048_BF16-NEXT: stp q3, q0, [x1, #96]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <64 x float>, ptr %a
+ %res = fptrunc <64 x float> %op1 to <64 x bfloat>
+ store <64 x bfloat> %res, ptr %b
+ ret void
+}
+
+;
+; FCVT D -> H
+;
+
+define void @fcvt_v1f64_v1bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ldr d0, [x0]
+; VBITS_GE_256_BASE-NEXT: mov w8, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: fcvtxn s0, d0
+; VBITS_GE_256_BASE-NEXT: fmov w9, s0
+; VBITS_GE_256_BASE-NEXT: ubfx w10, w9, #16, #1
+; VBITS_GE_256_BASE-NEXT: add w8, w9, w8
+; VBITS_GE_256_BASE-NEXT: add w8, w10, w8
+; VBITS_GE_256_BASE-NEXT: lsr w8, w8, #16
+; VBITS_GE_256_BASE-NEXT: fmov s0, w8
+; VBITS_GE_256_BASE-NEXT: str h0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ldr d0, [x0]
+; VBITS_GE_512_BASE-NEXT: mov w8, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: fcvtxn s0, d0
+; VBITS_GE_512_BASE-NEXT: fmov w9, s0
+; VBITS_GE_512_BASE-NEXT: ubfx w10, w9, #16, #1
+; VBITS_GE_512_BASE-NEXT: add w8, w9, w8
+; VBITS_GE_512_BASE-NEXT: add w8, w10, w8
+; VBITS_GE_512_BASE-NEXT: lsr w8, w8, #16
+; VBITS_GE_512_BASE-NEXT: fmov s0, w8
+; VBITS_GE_512_BASE-NEXT: str h0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ldr d0, [x0]
+; VBITS_GE_2048_BASE-NEXT: mov w8, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: fcvtxn s0, d0
+; VBITS_GE_2048_BASE-NEXT: fmov w9, s0
+; VBITS_GE_2048_BASE-NEXT: ubfx w10, w9, #16, #1
+; VBITS_GE_2048_BASE-NEXT: add w8, w9, w8
+; VBITS_GE_2048_BASE-NEXT: add w8, w10, w8
+; VBITS_GE_2048_BASE-NEXT: lsr w8, w8, #16
+; VBITS_GE_2048_BASE-NEXT: fmov s0, w8
+; VBITS_GE_2048_BASE-NEXT: str h0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_256_BF16-NEXT: fcvtxn s0, d0
+; VBITS_GE_256_BF16-NEXT: bfcvt h0, s0
+; VBITS_GE_256_BF16-NEXT: str h0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_512_BF16-NEXT: fcvtxn s0, d0
+; VBITS_GE_512_BF16-NEXT: bfcvt h0, s0
+; VBITS_GE_512_BF16-NEXT: str h0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v1f64_v1bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_2048_BF16-NEXT: fcvtxn s0, d0
+; VBITS_GE_2048_BF16-NEXT: bfcvt h0, s0
+; VBITS_GE_2048_BF16-NEXT: str h0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <1 x double>, ptr %a
+ %res = fptrunc <1 x double> %op1 to <1 x bfloat>
+ store <1 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v2f64_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d
+; VBITS_GE_256_BASE-NEXT: ldr q0, [x0]
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: str s0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d
+; VBITS_GE_512_BASE-NEXT: ldr q0, [x0]
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: str s0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d
+; VBITS_GE_2048_BASE-NEXT: ldr q0, [x0]
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: str s0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d
+; VBITS_GE_256_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_256_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: str s0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d
+; VBITS_GE_512_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_512_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: str s0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v2f64_v2bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d
+; VBITS_GE_2048_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_2048_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: str s0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <2 x double>, ptr %a
+ %res = fptrunc <2 x double> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f64_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d
+; VBITS_GE_256_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: str d0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d, vl4
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d
+; VBITS_GE_512_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: str d0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d, vl4
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d
+; VBITS_GE_2048_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: str d0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d
+; VBITS_GE_256_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: str d0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d, vl4
+; VBITS_GE_512_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d
+; VBITS_GE_512_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: str d0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v4f64_v4bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d, vl4
+; VBITS_GE_2048_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d
+; VBITS_GE_2048_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: str d0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <4 x double>, ptr %a
+ %res = fptrunc <4 x double> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f64_v8bf16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p1.d, vl4
+; VBITS_GE_256_BASE-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d
+; VBITS_GE_256_BASE-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3]
+; VBITS_GE_256_BASE-NEXT: ld1d { z1.d }, p1/z, [x0]
+; VBITS_GE_256_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: add z4.s, z0.s, z2.s
+; VBITS_GE_256_BASE-NEXT: add z2.s, z1.s, z2.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: add z3.s, z3.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z2.s, z5.s, z2.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_256_BASE-NEXT: sel z1.s, p0, z1.s, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: mov v1.d[1], v0.d[0]
+; VBITS_GE_256_BASE-NEXT: str q1, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d
+; VBITS_GE_512_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: str q0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d, vl8
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d
+; VBITS_GE_2048_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: str q0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d, vl4
+; VBITS_GE_256_BF16-NEXT: mov x8, #4 // =0x4
+; VBITS_GE_256_BF16-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; VBITS_GE_256_BF16-NEXT: ld1d { z1.d }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d
+; VBITS_GE_256_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: mov v1.d[1], v0.d[0]
+; VBITS_GE_256_BF16-NEXT: str q1, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d, vl8
+; VBITS_GE_512_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d
+; VBITS_GE_512_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: str q0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v8f64_v8bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d, vl8
+; VBITS_GE_2048_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d
+; VBITS_GE_2048_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: str q0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <8 x double>, ptr %a
+ %res = fptrunc <8 x double> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f64_v16bf16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d, vl16
+; VBITS_GE_256_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d
+; VBITS_GE_256_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_256_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_256_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z5.s, z0.s, z2.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: lsr z4.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z2.s, z1.s, z2.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_256_BASE-NEXT: add z2.s, z4.s, z2.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_256_BASE-NEXT: sel z1.s, p0, z1.s, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d, vl16
+; VBITS_GE_512_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d
+; VBITS_GE_512_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_512_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_512_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z5.s, z0.s, z2.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: lsr z4.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z2.s, z1.s, z2.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_512_BASE-NEXT: add z2.s, z4.s, z2.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_512_BASE-NEXT: sel z1.s, p0, z1.s, z2.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d, vl16
+; VBITS_GE_2048_BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d
+; VBITS_GE_2048_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_2048_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z5.s, z0.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: lsr z4.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z2.s, z1.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: add z2.s, z4.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: sel z1.s, p0, z1.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d, vl16
+; VBITS_GE_256_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d
+; VBITS_GE_256_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_256_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_256_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d, vl16
+; VBITS_GE_512_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_512_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v16f64_v16bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d, vl16
+; VBITS_GE_2048_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; VBITS_GE_2048_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <16 x double>, ptr %a
+ %res = fptrunc <16 x double> %op1 to <16 x bfloat>
+ store <16 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v32f64_v32bf16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d, vl32
+; VBITS_GE_256_BASE-NEXT: mov z4.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.d
+; VBITS_GE_256_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_256_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_256_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_256_BASE-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_256_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BASE-NEXT: fcvtx z3.s, p0/m, z1.d
+; VBITS_GE_256_BASE-NEXT: ext z1.b, z1.b, z1.b, #64
+; VBITS_GE_256_BASE-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z16.s, z0.s, z4.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BASE-NEXT: lsr z6.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z7.s, z3.s, #16
+; VBITS_GE_256_BASE-NEXT: add z17.s, z2.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z18.s, z3.s, z4.s
+; VBITS_GE_256_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_256_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: lsr z19.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z4.s, z1.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z5.s, z5.s, z16.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p3.s, p0/z, z3.s, z3.s
+; VBITS_GE_256_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z6.s, z6.s, z17.s
+; VBITS_GE_256_BASE-NEXT: add z7.s, z7.s, z18.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z5.s
+; VBITS_GE_256_BASE-NEXT: add z4.s, z19.s, z4.s
+; VBITS_GE_256_BASE-NEXT: sel z2.s, p2, z2.s, z6.s
+; VBITS_GE_256_BASE-NEXT: sel z3.s, p3, z3.s, z7.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: sel z1.s, p0, z1.s, z4.s
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_256_BASE-NEXT: stp q3, q1, [x1, #32]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d, vl32
+; VBITS_GE_512_BASE-NEXT: mov z4.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.d
+; VBITS_GE_512_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_512_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_512_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_512_BASE-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_512_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BASE-NEXT: fcvtx z3.s, p0/m, z1.d
+; VBITS_GE_512_BASE-NEXT: ext z1.b, z1.b, z1.b, #64
+; VBITS_GE_512_BASE-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_512_BASE-NEXT: lsr z5.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z16.s, z0.s, z4.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_512_BASE-NEXT: lsr z6.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z7.s, z3.s, #16
+; VBITS_GE_512_BASE-NEXT: add z17.s, z2.s, z4.s
+; VBITS_GE_512_BASE-NEXT: add z18.s, z3.s, z4.s
+; VBITS_GE_512_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_512_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_512_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: lsr z19.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_512_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z4.s, z1.s, z4.s
+; VBITS_GE_512_BASE-NEXT: add z5.s, z5.s, z16.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p3.s, p0/z, z3.s, z3.s
+; VBITS_GE_512_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z6.s, z6.s, z17.s
+; VBITS_GE_512_BASE-NEXT: add z7.s, z7.s, z18.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p1, z0.s, z5.s
+; VBITS_GE_512_BASE-NEXT: add z4.s, z19.s, z4.s
+; VBITS_GE_512_BASE-NEXT: sel z2.s, p2, z2.s, z6.s
+; VBITS_GE_512_BASE-NEXT: sel z3.s, p3, z3.s, z7.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: sel z1.s, p0, z1.s, z4.s
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_512_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_512_BASE-NEXT: stp q3, q1, [x1, #32]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d, vl32
+; VBITS_GE_2048_BASE-NEXT: mov z4.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.d
+; VBITS_GE_2048_BASE-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_2048_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BASE-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_2048_BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BASE-NEXT: fcvtx z3.s, p0/m, z1.d
+; VBITS_GE_2048_BASE-NEXT: ext z1.b, z1.b, z1.b, #64
+; VBITS_GE_2048_BASE-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_2048_BASE-NEXT: lsr z5.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z16.s, z0.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_2048_BASE-NEXT: lsr z6.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z7.s, z3.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z17.s, z2.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: add z18.s, z3.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: lsr z19.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z6.s, z6.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: and z7.s, z7.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z4.s, z1.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: add z5.s, z5.s, z16.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p3.s, p0/z, z3.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: orr z3.s, z3.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z19.s, z19.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z6.s, z6.s, z17.s
+; VBITS_GE_2048_BASE-NEXT: add z7.s, z7.s, z18.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: orr z1.s, z1.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p1, z0.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: add z4.s, z19.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: sel z2.s, p2, z2.s, z6.s
+; VBITS_GE_2048_BASE-NEXT: sel z3.s, p3, z3.s, z7.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: sel z1.s, p0, z1.s, z4.s
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z3.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: stp q0, q2, [x1]
+; VBITS_GE_2048_BASE-NEXT: stp q3, q1, [x1, #32]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d, vl32
+; VBITS_GE_256_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.d
+; VBITS_GE_256_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_256_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_256_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_256_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_256_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_256_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_256_BF16-NEXT: ext z3.b, z3.b, z1.b, #64
+; VBITS_GE_256_BF16-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_256_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: fcvtx z3.s, p0/m, z3.d
+; VBITS_GE_256_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_256_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_256_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d, vl32
+; VBITS_GE_512_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.d
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_512_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_512_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_512_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_512_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_512_BF16-NEXT: ext z3.b, z3.b, z1.b, #64
+; VBITS_GE_512_BF16-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_512_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: fcvtx z3.s, p0/m, z3.d
+; VBITS_GE_512_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_512_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v32f64_v32bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d, vl32
+; VBITS_GE_2048_BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.d
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; VBITS_GE_2048_BF16-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; VBITS_GE_2048_BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; VBITS_GE_2048_BF16-NEXT: movprfx z3, z1
+; VBITS_GE_2048_BF16-NEXT: ext z3.b, z3.b, z1.b, #64
+; VBITS_GE_2048_BF16-NEXT: fcvtx z2.s, p0/m, z2.d
+; VBITS_GE_2048_BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: fcvtx z3.s, p0/m, z3.d
+; VBITS_GE_2048_BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z2.s, z2.s, z2.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z3.s, z3.s, z3.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; VBITS_GE_2048_BF16-NEXT: stp q0, q2, [x1]
+; VBITS_GE_2048_BF16-NEXT: stp q1, q3, [x1, #32]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <32 x double>, ptr %a
+ %res = fptrunc <32 x double> %op1 to <32 x bfloat>
+ store <32 x bfloat> %res, ptr %b
+ ret void
+}
More information about the llvm-commits
mailing list