[llvm] [AArch64] Lower fixed width bf16 fpround (PR #209411)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 02:25:14 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: David Green (davemgreen)
<details>
<summary>Changes</summary>
Similar to #<!-- -->209194, this was previously hitting an error with an illegal FP_ROUND_MERGE_PASSTHRU. This time we lower the fptrunc to a scalable fptrunc and let it legalize naturally. On SVE systems a f64->bf16 fptrunc will fail to legalize, but lowers successfully with sve2 with fcvtn. A store(fptrunc) -> truncstore combine was disabled too as it fails with bf16.
---
Patch is 90.95 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/209411.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+4-5)
- (modified) llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll (+1862-6)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0a8a116098275..ab19d8e905aa1 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27362,11 +27362,12 @@ static SDValue performSTORECombine(SDNode *N,
// We purposefully don't care about legality of the nodes here as we know
// they can be split down into something legal.
if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND &&
- Value.getNode()->hasOneUse() && ST->isUnindexed() &&
+ Value->hasOneUse() && ST->isUnindexed() &&
Subtarget->useSVEForFixedLengthVectors() &&
ValueVT.isFixedLengthVector() &&
ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() &&
- hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()))
+ hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()) &&
+ Value.getValueType().getScalarType() != MVT::bf16)
return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT,
ST->getMemOperand());
@@ -33985,11 +33986,9 @@ AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
EVT ContainerSrcVT = getContainerForFixedLengthVector(DAG, SrcVT);
EVT RoundVT = ContainerSrcVT.changeVectorElementType(
*DAG.getContext(), VT.getVectorElementType());
- SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
- Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val,
- Op.getOperand(1), DAG.getPOISON(RoundVT));
+ Val = DAG.getNode(Op.getOpcode(), DL, RoundVT, Val, Op.getOperand(1));
Val = getSVESafeBitCast(ContainerSrcVT.changeTypeToInteger(), Val, DAG);
Val = convertFromScalableVector(DAG, SrcVT.changeTypeToInteger(), Val);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
index 98d6b7681bbf6..0ab8786387d31 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_256_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_512_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048,VBITS_GE_2048_BASE
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_256_BF16
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_512_BF16
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_2048,VBITS_GE_2048_BF16
define void @fcvt_v2bf16_v2f32(ptr %a, ptr %b) vscale_range(2,0) {
; CHECK-LABEL: fcvt_v2bf16_v2f32:
@@ -992,3 +992,1859 @@ define void @fcvt_v32bf16_v32f64(ptr %a, ptr %b) vscale_range(16,0) {
store <32 x double> %res, ptr %b
ret void
}
+
+
+define void @fcvt_v2f32_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_256_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_256_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_256_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_256_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_256_BASE-NEXT: str s0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_512_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_512_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_512_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_512_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_512_BASE-NEXT: str s0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ldr d1, [x0]
+; VBITS_GE_2048_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_2048_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_2048_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_2048_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_2048_BASE-NEXT: str s0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_256_BF16-NEXT: str s0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_512_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_512_BF16-NEXT: str s0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v2f32_v2bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ldr d0, [x0]
+; VBITS_GE_2048_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_2048_BF16-NEXT: str s0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <2 x float>, ptr %a
+ %res = fptrunc <2 x float> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f32_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_256_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_256_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_256_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_256_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_256_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_256_BASE-NEXT: str d0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_512_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_512_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_512_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_512_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_512_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_512_BASE-NEXT: str d0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ldr q1, [x0]
+; VBITS_GE_2048_BASE-NEXT: movi v0.4s, #127, msl #8
+; VBITS_GE_2048_BASE-NEXT: ushr v2.4s, v1.4s, #16
+; VBITS_GE_2048_BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: orr v1.4s, #64, lsl #16
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; VBITS_GE_2048_BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; VBITS_GE_2048_BASE-NEXT: shrn v0.4h, v0.4s, #16
+; VBITS_GE_2048_BASE-NEXT: str d0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_256_BF16-NEXT: str d0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_512_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_512_BF16-NEXT: str d0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v4f32_v4bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ldr q0, [x0]
+; VBITS_GE_2048_BF16-NEXT: bfcvtn v0.4h, v0.4s
+; VBITS_GE_2048_BF16-NEXT: str d0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <4 x float>, ptr %a
+ %res = fptrunc <4 x float> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f32_v8bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: str q0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: str q0, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: lsr z2.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z2.s, z2.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: str q0, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: str q0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: str q0, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v8f32_v8bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_2048_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BF16-NEXT: str q0, [x1]
+; VBITS_GE_2048_BF16-NEXT: ret
+ %op1 = load <8 x float>, ptr %a
+ %res = fptrunc <8 x float> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f32_v16bf16(ptr %a, ptr %b) #0 {
+; VBITS_GE_256_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_256_BASE: // %bb.0:
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BASE-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_256_BASE-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256_BASE-NEXT: ld1w { z2.s }, p0/z, [x0]
+; VBITS_GE_256_BASE-NEXT: ptrue p0.s
+; VBITS_GE_256_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z5.s, z2.s, #16
+; VBITS_GE_256_BASE-NEXT: add z4.s, z0.s, z1.s
+; VBITS_GE_256_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_256_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_256_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_256_BASE-NEXT: and z5.s, z5.s, #0x1
+; VBITS_GE_256_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_256_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_256_BASE-NEXT: add z3.s, z3.s, z4.s
+; VBITS_GE_256_BASE-NEXT: add z1.s, z5.s, z1.s
+; VBITS_GE_256_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_256_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_256_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_256_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_256_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BASE-NEXT: stp q1, q0, [x1]
+; VBITS_GE_256_BASE-NEXT: ret
+;
+; VBITS_GE_512_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_512_BASE: // %bb.0:
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_512_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BASE-NEXT: ptrue p0.s
+; VBITS_GE_512_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_512_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_512_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: add z5.s, z0.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_512_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: lsr z4.s, z2.s, #16
+; VBITS_GE_512_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_512_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_512_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_512_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_512_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_512_BASE-NEXT: add z1.s, z4.s, z1.s
+; VBITS_GE_512_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_512_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_512_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_512_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_512_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BASE-NEXT: ret
+;
+; VBITS_GE_2048_BASE-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_2048_BASE: // %bb.0:
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s, vl16
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; VBITS_GE_2048_BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BASE-NEXT: ptrue p0.s
+; VBITS_GE_2048_BASE-NEXT: movprfx z2, z0
+; VBITS_GE_2048_BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; VBITS_GE_2048_BASE-NEXT: lsr z3.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: add z5.s, z0.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; VBITS_GE_2048_BASE-NEXT: orr z0.s, z0.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: lsr z4.s, z2.s, #16
+; VBITS_GE_2048_BASE-NEXT: and z3.s, z3.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z2.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; VBITS_GE_2048_BASE-NEXT: orr z2.s, z2.s, #0x400000
+; VBITS_GE_2048_BASE-NEXT: and z4.s, z4.s, #0x1
+; VBITS_GE_2048_BASE-NEXT: add z3.s, z3.s, z5.s
+; VBITS_GE_2048_BASE-NEXT: add z1.s, z4.s, z1.s
+; VBITS_GE_2048_BASE-NEXT: sel z0.s, p1, z0.s, z3.s
+; VBITS_GE_2048_BASE-NEXT: mov z1.s, p0/m, z2.s
+; VBITS_GE_2048_BASE-NEXT: lsr z0.s, z0.s, #16
+; VBITS_GE_2048_BASE-NEXT: lsr z1.s, z1.s, #16
+; VBITS_GE_2048_BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_2048_BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_2048_BASE-NEXT: stp q0, q1, [x1]
+; VBITS_GE_2048_BASE-NEXT: ret
+;
+; VBITS_GE_256_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_256_BF16: // %bb.0:
+; VBITS_GE_256_BF16-NEXT: ptrue p0.s, vl8
+; VBITS_GE_256_BF16-NEXT: mov x8, #8 // =0x8
+; VBITS_GE_256_BF16-NEXT: ptrue p1.s
+; VBITS_GE_256_BF16-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; VBITS_GE_256_BF16-NEXT: ld1w { z1.s }, p0/z, [x0]
+; VBITS_GE_256_BF16-NEXT: bfcvt z0.h, p1/m, z0.s
+; VBITS_GE_256_BF16-NEXT: bfcvt z1.h, p1/m, z1.s
+; VBITS_GE_256_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_256_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_256_BF16-NEXT: stp q1, q0, [x1]
+; VBITS_GE_256_BF16-NEXT: ret
+;
+; VBITS_GE_512_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_512_BF16: // %bb.0:
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s, vl16
+; VBITS_GE_512_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_512_BF16-NEXT: ptrue p0.s
+; VBITS_GE_512_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_512_BF16-NEXT: ext z1.b, z1.b, z0.b, #32
+; VBITS_GE_512_BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; VBITS_GE_512_BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; VBITS_GE_512_BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; VBITS_GE_512_BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; VBITS_GE_512_BF16-NEXT: stp q0, q1, [x1]
+; VBITS_GE_512_BF16-NEXT: ret
+;
+; VBITS_GE_2048_BF16-LABEL: fcvt_v16f32_v16bf16:
+; VBITS_GE_2048_BF16: // %bb.0:
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s, vl16
+; VBITS_GE_2048_BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; VBITS_GE_2048_BF16-NEXT: ptrue p0.s
+; VBITS_GE_2048_BF16-NEXT: movprfx z1, z0
+; VBITS_GE_2048_BF16-NEXT: ext z1.b, z1.b, z0.b, #32
+; VBITS_GE_2048_BF1...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/209411
More information about the llvm-commits
mailing list