[llvm-branch-commits] [llvm] 9e60777 - [AArch64] Lower fixed width bf16 fpround (#209411)
Douglas Yung via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Jul 22 06:46:48 PDT 2026
Author: David Green
Date: 2026-07-22T13:46:34Z
New Revision: 9e6077709b2cb1ccc7dfde33112246b591920c6a
URL: https://github.com/llvm/llvm-project/commit/9e6077709b2cb1ccc7dfde33112246b591920c6a
DIFF: https://github.com/llvm/llvm-project/commit/9e6077709b2cb1ccc7dfde33112246b591920c6a.diff
LOG: [AArch64] Lower fixed width bf16 fpround (#209411)
Similar to #209194, this was previously hitting an error with an illegal
FP_ROUND_MERGE_PASSTHRU. This time we lower the fptrunc to a scalable
fptrunc and let it legalize naturally. On SVE systems a f64->bf16
fptrunc will fail to legalize, but lowers successfully with sve2 with
fcvtn. A store(fptrunc) -> truncstore combine was disabled too as it
fails with bf16.
(cherry picked from commit 50f2b9443d23eba757c3c0f42eef71e0f9721b15)
Added:
llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll
llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d6f2633297e51..cd516aae0365a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27379,11 +27379,6 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
- auto hasValidElementTypeForFPTruncStore = [](EVT VT) {
- EVT EltVT = VT.getVectorElementType();
- return EltVT == MVT::f32 || EltVT == MVT::f64;
- };
-
// Cast ptr32 and ptr64 pointers to the default address space before a store.
unsigned AddrSpace = ST->getAddressSpace();
if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR ||
@@ -27400,16 +27395,24 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget))
return Res;
+ auto hasValidElementTypeForFPTruncStore = [](EVT DstVT, EVT SrcVT) {
+ if (DstVT.getScalarType() == MVT::bf16)
+ return false;
+ EVT EltVT = SrcVT.getVectorElementType();
+ return EltVT == MVT::f32 || EltVT == MVT::f64;
+ };
+
// If this is an FP_ROUND followed by a store, fold this into a truncating
// store. We can do this even if this is already a truncstore.
// We purposefully don't care about legality of the nodes here as we know
// they can be split down into something legal.
if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND &&
- Value.getNode()->hasOneUse() && ST->isUnindexed() &&
+ Value->hasOneUse() && ST->isUnindexed() &&
Subtarget->useSVEForFixedLengthVectors() &&
ValueVT.isFixedLengthVector() &&
ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() &&
- hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()))
+ hasValidElementTypeForFPTruncStore(Value.getValueType(),
+ Value.getOperand(0).getValueType()))
return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT,
ST->getMemOperand());
@@ -34028,11 +34031,9 @@ AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
EVT ContainerSrcVT = getContainerForFixedLengthVector(DAG, SrcVT);
EVT RoundVT = ContainerSrcVT.changeVectorElementType(
*DAG.getContext(), VT.getVectorElementType());
- SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
- Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val,
- Op.getOperand(1), DAG.getPOISON(RoundVT));
+ Val = DAG.getNode(Op.getOpcode(), DL, RoundVT, Val, Op.getOperand(1));
Val = getSVESafeBitCast(ContainerSrcVT.changeTypeToInteger(), Val, DAG);
Val = convertFromScalableVector(DAG, SrcVT.changeTypeToInteger(), Val);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
rename to llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll
new file mode 100644
index 0000000000000..6d988dadf64e4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll
@@ -0,0 +1,840 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_256
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_512
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_2048
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_256
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_512
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_2048
+
+define void @fcvt_v2f32_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v2f32_v2bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr d1, [x0]
+; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8
+; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16
+; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-BASE-NEXT: str s0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v2f32_v2bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ldr d0, [x0]
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: str s0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <2 x float>, ptr %a
+ %res = fptrunc <2 x float> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f32_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v4f32_v4bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr q1, [x0]
+; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8
+; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16
+; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-BASE-NEXT: str d0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v4f32_v4bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ldr q0, [x0]
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: str d0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <4 x float>, ptr %a
+ %res = fptrunc <4 x float> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f32_v8bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v8f32_v8bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl8
+; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16
+; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s
+; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: str q0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v8f32_v8bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.s, vl8
+; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.s
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: str q0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <8 x float>, ptr %a
+ %res = fptrunc <8 x float> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f32_v16bf16(ptr %a, ptr %b) #0 {
+; CHECK_BASE_GE_256-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_256: // %bb.0:
+; CHECK_BASE_GE_256-NEXT: ptrue p0.s, vl8
+; CHECK_BASE_GE_256-NEXT: mov x8, #8 // =0x8
+; CHECK_BASE_GE_256-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK_BASE_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; CHECK_BASE_GE_256-NEXT: ptrue p0.s
+; CHECK_BASE_GE_256-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z5.s, z2.s, #16
+; CHECK_BASE_GE_256-NEXT: add z4.s, z0.s, z1.s
+; CHECK_BASE_GE_256-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_256-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_256-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_256-NEXT: and z5.s, z5.s, #0x1
+; CHECK_BASE_GE_256-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_256-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: add z3.s, z3.s, z4.s
+; CHECK_BASE_GE_256-NEXT: add z1.s, z5.s, z1.s
+; CHECK_BASE_GE_256-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_256-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_256-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_256-NEXT: stp q1, q0, [x1]
+; CHECK_BASE_GE_256-NEXT: ret
+;
+; CHECK_BASE_GE_512-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_512: // %bb.0:
+; CHECK_BASE_GE_512-NEXT: ptrue p0.s, vl16
+; CHECK_BASE_GE_512-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BASE_GE_512-NEXT: ptrue p0.s
+; CHECK_BASE_GE_512-NEXT: movprfx z2, z0
+; CHECK_BASE_GE_512-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK_BASE_GE_512-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: add z5.s, z0.s, z1.s
+; CHECK_BASE_GE_512-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_512-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_512-NEXT: lsr z4.s, z2.s, #16
+; CHECK_BASE_GE_512-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_512-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_512-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_512-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_512-NEXT: and z4.s, z4.s, #0x1
+; CHECK_BASE_GE_512-NEXT: add z3.s, z3.s, z5.s
+; CHECK_BASE_GE_512-NEXT: add z1.s, z4.s, z1.s
+; CHECK_BASE_GE_512-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_512-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_512-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_512-NEXT: stp q0, q1, [x1]
+; CHECK_BASE_GE_512-NEXT: ret
+;
+; CHECK_BASE_GE_2048-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_2048: // %bb.0:
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.s, vl16
+; CHECK_BASE_GE_2048-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.s
+; CHECK_BASE_GE_2048-NEXT: movprfx z2, z0
+; CHECK_BASE_GE_2048-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK_BASE_GE_2048-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: add z5.s, z0.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_2048-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_2048-NEXT: lsr z4.s, z2.s, #16
+; CHECK_BASE_GE_2048-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_2048-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_2048-NEXT: and z4.s, z4.s, #0x1
+; CHECK_BASE_GE_2048-NEXT: add z3.s, z3.s, z5.s
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z4.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_2048-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_2048-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_2048-NEXT: stp q0, q1, [x1]
+; CHECK_BASE_GE_2048-NEXT: ret
+;
+; CHECK_BF16_GE_256-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_256: // %bb.0:
+; CHECK_BF16_GE_256-NEXT: ptrue p0.s, vl8
+; CHECK_BF16_GE_256-NEXT: mov x8, #8 // =0x8
+; CHECK_BF16_GE_256-NEXT: ptrue p1.s
+; CHECK_BF16_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK_BF16_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK_BF16_GE_256-NEXT: bfcvt z0.h, p1/m, z0.s
+; CHECK_BF16_GE_256-NEXT: bfcvt z1.h, p1/m, z1.s
+; CHECK_BF16_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_256-NEXT: stp q1, q0, [x1]
+; CHECK_BF16_GE_256-NEXT: ret
+;
+; CHECK_BF16_GE_512-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_512: // %bb.0:
+; CHECK_BF16_GE_512-NEXT: ptrue p0.s, vl16
+; CHECK_BF16_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BF16_GE_512-NEXT: ptrue p0.s
+; CHECK_BF16_GE_512-NEXT: movprfx z1, z0
+; CHECK_BF16_GE_512-NEXT: ext z1.b, z1.b, z0.b, #32
+; CHECK_BF16_GE_512-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_512-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK_BF16_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_512-NEXT: stp q0, q1, [x1]
+; CHECK_BF16_GE_512-NEXT: ret
+;
+; CHECK_BF16_GE_2048-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_2048: // %bb.0:
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.s, vl16
+; CHECK_BF16_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.s
+; CHECK_BF16_GE_2048-NEXT: movprfx z1, z0
+; CHECK_BF16_GE_2048-NEXT: ext z1.b, z1.b, z0.b, #32
+; CHECK_BF16_GE_2048-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_2048-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK_BF16_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_2048-NEXT: stp q0, q1, [x1]
+; CHECK_BF16_GE_2048-NEXT: ret
+ %op1 = load <16 x float>, ptr %a
+ %res = fptrunc <16 x float> %op1 to <16 x bfloat>
+ store <16 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v32f32_v32bf16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v32f32_v32bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl32
+; CHECK-BASE-NEXT: mov z3.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BASE-NEXT: movprfx z2, z0
+; CHECK-BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK-BASE-NEXT: lsr z4.s, z0.s, #16
+; CHECK-BASE-NEXT: add z16.s, z0.s, z3.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: movprfx z7, z1
+; CHECK-BASE-NEXT: ext z7.b, z7.b, z1.b, #32
+; CHECK-BASE-NEXT: lsr z5.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z6.s, z1.s, #16
+; CHECK-BASE-NEXT: add z17.s, z2.s, z3.s
+; CHECK-BASE-NEXT: add z18.s, z1.s, z3.s
+; CHECK-BASE-NEXT: and z4.s, z4.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-BASE-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK-BASE-NEXT: lsr z19.s, z7.s, #16
+; CHECK-BASE-NEXT: and z5.s, z5.s, #0x1
+; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1
+; CHECK-BASE-NEXT: add z3.s, z7.s, z3.s
+; CHECK-BASE-NEXT: add z4.s, z4.s, z16.s
+; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: and z19.s, z19.s, #0x1
+; CHECK-BASE-NEXT: add z5.s, z5.s, z17.s
+; CHECK-BASE-NEXT: add z6.s, z6.s, z18.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s
+; CHECK-BASE-NEXT: orr z7.s, z7.s, #0x400000
+; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z4.s
+; CHECK-BASE-NEXT: add z3.s, z19.s, z3.s
+; CHECK-BASE-NEXT: sel z2.s, p2, z2.s, z5.s
+; CHECK-BASE-NEXT: sel z1.s, p3, z1.s, z6.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: mov z3.s, p0/m, z7.s
+; CHECK-BASE-NEXT: lsr z2.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16
+; CHECK-BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BASE-NEXT: stp q0, q2, [x1]
+; CHECK-BASE-NEXT: stp q1, q3, [x1, #32]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v32f32_v32bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.s, vl32
+; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.s
+; CHECK-BF16-NEXT: movprfx z1, z0
+; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BF16-NEXT: movprfx z2, z0
+; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: movprfx z3, z1
+; CHECK-BF16-NEXT: ext z3.b, z3.b, z1.b, #32
+; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BF16-NEXT: stp q0, q2, [x1]
+; CHECK-BF16-NEXT: stp q1, q3, [x1, #32]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <32 x float>, ptr %a
+ %res = fptrunc <32 x float> %op1 to <32 x bfloat>
+ store <32 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v64f32_v64bf16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v64f32_v64bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl64
+; CHECK-BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; CHECK-BASE-NEXT: movprfx z3, z0
+; CHECK-BASE-NEXT: ext z3.b, z3.b, z0.b, #32
+; CHECK-BASE-NEXT: movprfx z4, z0
+; CHECK-BASE-NEXT: ext z4.b, z4.b, z0.b, #64
+; CHECK-BASE-NEXT: lsr z16.s, z0.s, #16
+; CHECK-BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: lsr z22.s, z1.s, #16
+; CHECK-BASE-NEXT: movprfx z5, z1
+; CHECK-BASE-NEXT: ext z5.b, z5.b, z1.b, #32
+; CHECK-BASE-NEXT: lsr z6.s, z3.s, #16
+; CHECK-BASE-NEXT: lsr z7.s, z4.s, #16
+; CHECK-BASE-NEXT: movprfx z17, z1
+; CHECK-BASE-NEXT: ext z17.b, z17.b, z1.b, #64
+; CHECK-BASE-NEXT: add z23.s, z1.s, z2.s
+; CHECK-BASE-NEXT: movprfx z18, z4
+; CHECK-BASE-NEXT: ext z18.b, z18.b, z4.b, #32
+; CHECK-BASE-NEXT: add z19.s, z3.s, z2.s
+; CHECK-BASE-NEXT: add z21.s, z4.s, z2.s
+; CHECK-BASE-NEXT: and z22.s, z22.s, #0x1
+; CHECK-BASE-NEXT: lsr z20.s, z5.s, #16
+; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1
+; CHECK-BASE-NEXT: and z7.s, z7.s, #0x1
+; CHECK-BASE-NEXT: movprfx z24, z17
+; CHECK-BASE-NEXT: ext z24.b, z24.b, z17.b, #32
+; CHECK-BASE-NEXT: and z16.s, z16.s, #0x1
+; CHECK-BASE-NEXT: add z26.s, z5.s, z2.s
+; CHECK-BASE-NEXT: lsr z25.s, z18.s, #16
+; CHECK-BASE-NEXT: add z22.s, z22.s, z23.s
+; CHECK-BASE-NEXT: add z23.s, z0.s, z2.s
+; CHECK-BASE-NEXT: and z20.s, z20.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s
+; CHECK-BASE-NEXT: add z6.s, z6.s, z19.s
+; CHECK-BASE-NEXT: add z7.s, z7.s, z21.s
+; CHECK-BASE-NEXT: lsr z21.s, z17.s, #16
+; CHECK-BASE-NEXT: orr z3.s, z3.s, #0x400000
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: add z16.s, z16.s, z23.s
+; CHECK-BASE-NEXT: lsr z23.s, z24.s, #16
+; CHECK-BASE-NEXT: add z20.s, z20.s, z26.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s
+; CHECK-BASE-NEXT: orr z5.s, z5.s, #0x400000
+; CHECK-BASE-NEXT: add z19.s, z18.s, z2.s
+; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s
+; CHECK-BASE-NEXT: add z26.s, z17.s, z2.s
+; CHECK-BASE-NEXT: and z25.s, z25.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: and z21.s, z21.s, #0x1
+; CHECK-BASE-NEXT: orr z4.s, z4.s, #0x400000
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: sel z3.s, p2, z3.s, z6.s
+; CHECK-BASE-NEXT: sel z0.s, p5, z0.s, z16.s
+; CHECK-BASE-NEXT: and z23.s, z23.s, #0x1
+; CHECK-BASE-NEXT: add z2.s, z24.s, z2.s
+; CHECK-BASE-NEXT: sel z5.s, p1, z5.s, z20.s
+; CHECK-BASE-NEXT: sel z4.s, p3, z4.s, z7.s
+; CHECK-BASE-NEXT: add z6.s, z25.s, z19.s
+; CHECK-BASE-NEXT: add z7.s, z21.s, z26.s
+; CHECK-BASE-NEXT: sel z1.s, p4, z1.s, z22.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16
+; CHECK-BASE-NEXT: add z2.s, z23.s, z2.s
+; CHECK-BASE-NEXT: lsr z5.s, z5.s, #16
+; CHECK-BASE-NEXT: lsr z4.s, z4.s, #16
+; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BASE-NEXT: uzp1 z5.h, z5.h, z5.h
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z18.s, z18.s
+; CHECK-BASE-NEXT: orr z18.s, z18.s, #0x400000
+; CHECK-BASE-NEXT: stp q0, q3, [x1]
+; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z17.s, z17.s
+; CHECK-BASE-NEXT: orr z17.s, z17.s, #0x400000
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z24.s, z24.s
+; CHECK-BASE-NEXT: orr z24.s, z24.s, #0x400000
+; CHECK-BASE-NEXT: stp q1, q5, [x1, #64]
+; CHECK-BASE-NEXT: mov z6.s, p1/m, z18.s
+; CHECK-BASE-NEXT: mov z7.s, p2/m, z17.s
+; CHECK-BASE-NEXT: mov z2.s, p0/m, z24.s
+; CHECK-BASE-NEXT: lsr z6.s, z6.s, #16
+; CHECK-BASE-NEXT: lsr z7.s, z7.s, #16
+; CHECK-BASE-NEXT: lsr z0.s, z2.s, #16
+; CHECK-BASE-NEXT: uzp1 z2.h, z4.h, z4.h
+; CHECK-BASE-NEXT: uzp1 z3.h, z6.h, z6.h
+; CHECK-BASE-NEXT: uzp1 z1.h, z7.h, z7.h
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: stp q2, q3, [x1, #32]
+; CHECK-BASE-NEXT: stp q1, q0, [x1, #96]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v64f32_v64bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.s, vl64
+; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.s
+; CHECK-BF16-NEXT: movprfx z3, z0
+; CHECK-BF16-NEXT: ext z3.b, z3.b, z0.b, #32
+; CHECK-BF16-NEXT: movprfx z1, z0
+; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; CHECK-BF16-NEXT: movprfx z2, z0
+; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; CHECK-BF16-NEXT: movprfx z4, z1
+; CHECK-BF16-NEXT: ext z4.b, z4.b, z1.b, #32
+; CHECK-BF16-NEXT: movprfx z5, z1
+; CHECK-BF16-NEXT: ext z5.b, z5.b, z1.b, #64
+; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-BF16-NEXT: movprfx z6, z2
+; CHECK-BF16-NEXT: ext z6.b, z6.b, z2.b, #32
+; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: bfcvt z4.h, p0/m, z4.s
+; CHECK-BF16-NEXT: movprfx z7, z5
+; CHECK-BF16-NEXT: ext z7.b, z7.b, z5.b, #32
+; CHECK-BF16-NEXT: bfcvt z5.h, p0/m, z5.s
+; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BF16-NEXT: bfcvt z6.h, p0/m, z6.s
+; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BF16-NEXT: stp q0, q3, [x1]
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z7.s
+; CHECK-BF16-NEXT: uzp1 z4.h, z4.h, z4.h
+; CHECK-BF16-NEXT: uzp1 z3.h, z5.h, z5.h
+; CHECK-BF16-NEXT: stp q1, q4, [x1, #64]
+; CHECK-BF16-NEXT: uzp1 z1.h, z6.h, z6.h
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: stp q2, q1, [x1, #32]
+; CHECK-BF16-NEXT: stp q3, q0, [x1, #96]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <64 x float>, ptr %a
+ %res = fptrunc <64 x float> %op1 to <64 x bfloat>
+ store <64 x bfloat> %res, ptr %b
+ ret void
+}
+
+; FCVT D -> H
+
+define void @fcvt_v1f64_v1bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v1f64_v1bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr d0, [x0]
+; CHECK-BASE-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-BASE-NEXT: fcvtxn s0, d0
+; CHECK-BASE-NEXT: fmov w9, s0
+; CHECK-BASE-NEXT: ubfx w10, w9, #16, #1
+; CHECK-BASE-NEXT: add w8, w9, w8
+; CHECK-BASE-NEXT: add w8, w10, w8
+; CHECK-BASE-NEXT: lsr w8, w8, #16
+; CHECK-BASE-NEXT: fmov s0, w8
+; CHECK-BASE-NEXT: str h0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v1f64_v1bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ldr d0, [x0]
+; CHECK-BF16-NEXT: fcvtxn s0, d0
+; CHECK-BF16-NEXT: bfcvt h0, s0
+; CHECK-BF16-NEXT: str h0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <1 x double>, ptr %a
+ %res = fptrunc <1 x double> %op1 to <1 x bfloat>
+ store <1 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v2f64_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v2f64_v2bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.d
+; CHECK-BASE-NEXT: ldr q0, [x0]
+; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16
+; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s
+; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: str s0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v2f64_v2bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.d
+; CHECK-BF16-NEXT: ldr q0, [x0]
+; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: str s0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <2 x double>, ptr %a
+ %res = fptrunc <2 x double> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f64_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v4f64_v4bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.d, vl4
+; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.d
+; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16
+; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s
+; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: str d0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v4f64_v4bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.d, vl4
+; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.d
+; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: str d0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <4 x double>, ptr %a
+ %res = fptrunc <4 x double> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f64_v8bf16(ptr %a, ptr %b) #0 {
+; CHECK_BASE_GE_256-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BASE_GE_256: // %bb.0:
+; CHECK_BASE_GE_256-NEXT: ptrue p1.d, vl4
+; CHECK_BASE_GE_256-NEXT: mov x8, #4 // =0x4
+; CHECK_BASE_GE_256-NEXT: mov z2.s, #32767 // =0x7fff
+; CHECK_BASE_GE_256-NEXT: ptrue p0.d
+; CHECK_BASE_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3]
+; CHECK_BASE_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0]
+; CHECK_BASE_GE_256-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BASE_GE_256-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK_BASE_GE_256-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z5.s, z1.s, #16
+; CHECK_BASE_GE_256-NEXT: add z4.s, z0.s, z2.s
+; CHECK_BASE_GE_256-NEXT: add z2.s, z1.s, z2.s
+; CHECK_BASE_GE_256-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_256-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_256-NEXT: and z5.s, z5.s, #0x1
+; CHECK_BASE_GE_256-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK_BASE_GE_256-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: add z3.s, z3.s, z4.s
+; CHECK_BASE_GE_256-NEXT: add z2.s, z5.s, z2.s
+; CHECK_BASE_GE_256-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_256-NEXT: sel z1.s, p0, z1.s, z2.s
+; CHECK_BASE_GE_256-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BASE_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK_BASE_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_256-NEXT: mov v1.d[1], v0.d[0]
+; CHECK_BASE_GE_256-NEXT: str q1, [x1]
+; CHECK_BASE_GE_256-NEXT: ret
+;
+; CHECK_BASE_GE_512-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BASE_GE_512: // %bb.0:
+; CHECK_BASE_GE_512-NEXT: ptrue p0.d, vl8
+; CHECK_BASE_GE_512-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK_BASE_GE_512-NEXT: ptrue p0.d
+; CHECK_BASE_GE_512-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BASE_GE_512-NEXT: lsr z2.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: add z1.s, z0.s, z1.s
+; CHECK_BASE_GE_512-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_512-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_512-NEXT: and z2.s, z2.s, #0x1
+; CHECK_BASE_GE_512-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK_BASE_GE_512-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BASE_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_512-NEXT: str q0, [x1]
+; CHECK_BASE_GE_512-NEXT: ret
+;
+; CHECK_BASE_GE_2048-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BASE_GE_2048: // %bb.0:
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.d, vl8
+; CHECK_BASE_GE_2048-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.d
+; CHECK_BASE_GE_2048-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BASE_GE_2048-NEXT: lsr z2.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z0.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_2048-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_2048-NEXT: and z2.s, z2.s, #0x1
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BASE_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_2048-NEXT: str q0, [x1]
+; CHECK_BASE_GE_2048-NEXT: ret
+;
+; CHECK_BF16_GE_256-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BF16_GE_256: // %bb.0:
+; CHECK_BF16_GE_256-NEXT: ptrue p0.d, vl4
+; CHECK_BF16_GE_256-NEXT: mov x8, #4 // =0x4
+; CHECK_BF16_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
+; CHECK_BF16_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK_BF16_GE_256-NEXT: ptrue p0.d
+; CHECK_BF16_GE_256-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BF16_GE_256-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK_BF16_GE_256-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_256-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK_BF16_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BF16_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK_BF16_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_256-NEXT: mov v1.d[1], v0.d[0]
+; CHECK_BF16_GE_256-NEXT: str q1, [x1]
+; CHECK_BF16_GE_256-NEXT: ret
+;
+; CHECK_BF16_GE_512-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BF16_GE_512: // %bb.0:
+; CHECK_BF16_GE_512-NEXT: ptrue p0.d, vl8
+; CHECK_BF16_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK_BF16_GE_512-NEXT: ptrue p0.d
+; CHECK_BF16_GE_512-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BF16_GE_512-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BF16_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_512-NEXT: str q0, [x1]
+; CHECK_BF16_GE_512-NEXT: ret
+;
+; CHECK_BF16_GE_2048-LABEL: fcvt_v8f64_v8bf16:
+; CHECK_BF16_GE_2048: // %bb.0:
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.d, vl8
+; CHECK_BF16_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.d
+; CHECK_BF16_GE_2048-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK_BF16_GE_2048-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK_BF16_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_2048-NEXT: str q0, [x1]
+; CHECK_BF16_GE_2048-NEXT: ret
+ %op1 = load <8 x double>, ptr %a
+ %res = fptrunc <8 x double> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f64_v16bf16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v16f64_v16bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.d, vl16
+; CHECK-BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.d
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-BASE-NEXT: lsr z3.s, z0.s, #16
+; CHECK-BASE-NEXT: add z5.s, z0.s, z2.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: lsr z4.s, z1.s, #16
+; CHECK-BASE-NEXT: and z3.s, z3.s, #0x1
+; CHECK-BASE-NEXT: add z2.s, z1.s, z2.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: and z4.s, z4.s, #0x1
+; CHECK-BASE-NEXT: add z3.s, z3.s, z5.s
+; CHECK-BASE-NEXT: add z2.s, z4.s, z2.s
+; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK-BASE-NEXT: sel z1.s, p0, z1.s, z2.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BASE-NEXT: stp q0, q1, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v16f64_v16bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.d, vl16
+; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.d
+; CHECK-BF16-NEXT: movprfx z1, z0
+; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BF16-NEXT: stp q0, q1, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <16 x double>, ptr %a
+ %res = fptrunc <16 x double> %op1 to <16 x bfloat>
+ store <16 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v32f64_v32bf16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v32f64_v32bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.d, vl32
+; CHECK-BASE-NEXT: mov z4.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.d
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; CHECK-BASE-NEXT: movprfx z2, z0
+; CHECK-BASE-NEXT: ext z2.b, z2.b, z0.b, #64
+; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BASE-NEXT: fcvtx z3.s, p0/m, z1.d
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z1.b, #64
+; CHECK-BASE-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-BASE-NEXT: lsr z5.s, z0.s, #16
+; CHECK-BASE-NEXT: add z16.s, z0.s, z4.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-BASE-NEXT: lsr z6.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z7.s, z3.s, #16
+; CHECK-BASE-NEXT: add z17.s, z2.s, z4.s
+; CHECK-BASE-NEXT: add z18.s, z3.s, z4.s
+; CHECK-BASE-NEXT: and z5.s, z5.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-BASE-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK-BASE-NEXT: lsr z19.s, z1.s, #16
+; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1
+; CHECK-BASE-NEXT: and z7.s, z7.s, #0x1
+; CHECK-BASE-NEXT: add z4.s, z1.s, z4.s
+; CHECK-BASE-NEXT: add z5.s, z5.s, z16.s
+; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z3.s, z3.s
+; CHECK-BASE-NEXT: orr z3.s, z3.s, #0x400000
+; CHECK-BASE-NEXT: and z19.s, z19.s, #0x1
+; CHECK-BASE-NEXT: add z6.s, z6.s, z17.s
+; CHECK-BASE-NEXT: add z7.s, z7.s, z18.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z5.s
+; CHECK-BASE-NEXT: add z4.s, z19.s, z4.s
+; CHECK-BASE-NEXT: sel z2.s, p2, z2.s, z6.s
+; CHECK-BASE-NEXT: sel z3.s, p3, z3.s, z7.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: sel z1.s, p0, z1.s, z4.s
+; CHECK-BASE-NEXT: lsr z2.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16
+; CHECK-BASE-NEXT: uzp1 z2.s, z2.s, z2.s
+; CHECK-BASE-NEXT: uzp1 z3.s, z3.s, z3.s
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK-BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BASE-NEXT: stp q0, q2, [x1]
+; CHECK-BASE-NEXT: stp q3, q1, [x1, #32]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v32f64_v32bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.d, vl32
+; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.d
+; CHECK-BF16-NEXT: movprfx z1, z0
+; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #128
+; CHECK-BF16-NEXT: movprfx z2, z0
+; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #64
+; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d
+; CHECK-BF16-NEXT: movprfx z3, z1
+; CHECK-BF16-NEXT: ext z3.b, z3.b, z1.b, #64
+; CHECK-BF16-NEXT: fcvtx z2.s, p0/m, z2.d
+; CHECK-BF16-NEXT: fcvtx z1.s, p0/m, z1.d
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: fcvtx z3.s, p0/m, z3.d
+; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s
+; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; CHECK-BF16-NEXT: uzp1 z2.s, z2.s, z2.s
+; CHECK-BF16-NEXT: uzp1 z1.s, z1.s, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: uzp1 z3.s, z3.s, z3.s
+; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BF16-NEXT: stp q0, q2, [x1]
+; CHECK-BF16-NEXT: stp q1, q3, [x1, #32]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <32 x double>, ptr %a
+ %res = fptrunc <32 x double> %op1 to <32 x bfloat>
+ store <32 x bfloat> %res, ptr %b
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-branch-commits
mailing list