[llvm-branch-commits] [llvm] release/23.x: [AArch64] Lower fixed width bf16 fpround (#209411) (PR #210264)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 17 00:57:59 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: llvmbot
<details>
<summary>Changes</summary>
Backport 50f2b9443d23eba757c3c0f42eef71e0f9721b15
Requested by: @<!-- -->davemgreen
---
Patch is 39.19 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/210264.diff
3 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+11-10)
- (renamed) llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll ()
- (added) llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll (+840)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d6f2633297e51..cd516aae0365a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27379,11 +27379,6 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget))
return Res;
- auto hasValidElementTypeForFPTruncStore = [](EVT VT) {
- EVT EltVT = VT.getVectorElementType();
- return EltVT == MVT::f32 || EltVT == MVT::f64;
- };
-
// Cast ptr32 and ptr64 pointers to the default address space before a store.
unsigned AddrSpace = ST->getAddressSpace();
if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR ||
@@ -27400,16 +27395,24 @@ static SDValue performSTORECombine(SDNode *N,
if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget))
return Res;
+ auto hasValidElementTypeForFPTruncStore = [](EVT DstVT, EVT SrcVT) {
+ if (DstVT.getScalarType() == MVT::bf16)
+ return false;
+ EVT EltVT = SrcVT.getVectorElementType();
+ return EltVT == MVT::f32 || EltVT == MVT::f64;
+ };
+
// If this is an FP_ROUND followed by a store, fold this into a truncating
// store. We can do this even if this is already a truncstore.
// We purposefully don't care about legality of the nodes here as we know
// they can be split down into something legal.
if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND &&
- Value.getNode()->hasOneUse() && ST->isUnindexed() &&
+ Value->hasOneUse() && ST->isUnindexed() &&
Subtarget->useSVEForFixedLengthVectors() &&
ValueVT.isFixedLengthVector() &&
ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() &&
- hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType()))
+ hasValidElementTypeForFPTruncStore(Value.getValueType(),
+ Value.getOperand(0).getValueType()))
return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT,
ST->getMemOperand());
@@ -34028,11 +34031,9 @@ AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op,
EVT ContainerSrcVT = getContainerForFixedLengthVector(DAG, SrcVT);
EVT RoundVT = ContainerSrcVT.changeVectorElementType(
*DAG.getContext(), VT.getVectorElementType());
- SDValue Pg = getPredicateForVector(DAG, DL, RoundVT);
Val = convertToScalableVector(DAG, ContainerSrcVT, Val);
- Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val,
- Op.getOperand(1), DAG.getPOISON(RoundVT));
+ Val = DAG.getNode(Op.getOpcode(), DL, RoundVT, Val, Op.getOperand(1));
Val = getSVESafeBitCast(ContainerSrcVT.changeTypeToInteger(), Val, DAG);
Val = convertFromScalableVector(DAG, SrcVT.changeTypeToInteger(), Val);
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll
rename to llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll
new file mode 100644
index 0000000000000..6d988dadf64e4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll
@@ -0,0 +1,840 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_256
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_512
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_2048
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_256
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_512
+; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_2048
+
+define void @fcvt_v2f32_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v2f32_v2bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr d1, [x0]
+; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8
+; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16
+; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-BASE-NEXT: str s0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v2f32_v2bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ldr d0, [x0]
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: str s0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <2 x float>, ptr %a
+ %res = fptrunc <2 x float> %op1 to <2 x bfloat>
+ store <2 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v4f32_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v4f32_v4bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ldr q1, [x0]
+; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8
+; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16
+; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s
+; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s
+; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b
+; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16
+; CHECK-BASE-NEXT: str d0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v4f32_v4bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ldr q0, [x0]
+; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; CHECK-BF16-NEXT: str d0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <4 x float>, ptr %a
+ %res = fptrunc <4 x float> %op1 to <4 x bfloat>
+ store <4 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v8f32_v8bf16(ptr %a, ptr %b) vscale_range(2,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v8f32_v8bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl8
+; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16
+; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1
+; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s
+; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: str q0, [x1]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v8f32_v8bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.s, vl8
+; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.s
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: str q0, [x1]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <8 x float>, ptr %a
+ %res = fptrunc <8 x float> %op1 to <8 x bfloat>
+ store <8 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v16f32_v16bf16(ptr %a, ptr %b) #0 {
+; CHECK_BASE_GE_256-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_256: // %bb.0:
+; CHECK_BASE_GE_256-NEXT: ptrue p0.s, vl8
+; CHECK_BASE_GE_256-NEXT: mov x8, #8 // =0x8
+; CHECK_BASE_GE_256-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK_BASE_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0]
+; CHECK_BASE_GE_256-NEXT: ptrue p0.s
+; CHECK_BASE_GE_256-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z5.s, z2.s, #16
+; CHECK_BASE_GE_256-NEXT: add z4.s, z0.s, z1.s
+; CHECK_BASE_GE_256-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_256-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_256-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_256-NEXT: and z5.s, z5.s, #0x1
+; CHECK_BASE_GE_256-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_256-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_256-NEXT: add z3.s, z3.s, z4.s
+; CHECK_BASE_GE_256-NEXT: add z1.s, z5.s, z1.s
+; CHECK_BASE_GE_256-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_256-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_256-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_256-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_256-NEXT: stp q1, q0, [x1]
+; CHECK_BASE_GE_256-NEXT: ret
+;
+; CHECK_BASE_GE_512-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_512: // %bb.0:
+; CHECK_BASE_GE_512-NEXT: ptrue p0.s, vl16
+; CHECK_BASE_GE_512-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BASE_GE_512-NEXT: ptrue p0.s
+; CHECK_BASE_GE_512-NEXT: movprfx z2, z0
+; CHECK_BASE_GE_512-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK_BASE_GE_512-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: add z5.s, z0.s, z1.s
+; CHECK_BASE_GE_512-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_512-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_512-NEXT: lsr z4.s, z2.s, #16
+; CHECK_BASE_GE_512-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_512-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_512-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_512-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_512-NEXT: and z4.s, z4.s, #0x1
+; CHECK_BASE_GE_512-NEXT: add z3.s, z3.s, z5.s
+; CHECK_BASE_GE_512-NEXT: add z1.s, z4.s, z1.s
+; CHECK_BASE_GE_512-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_512-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_512-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_512-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_512-NEXT: stp q0, q1, [x1]
+; CHECK_BASE_GE_512-NEXT: ret
+;
+; CHECK_BASE_GE_2048-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BASE_GE_2048: // %bb.0:
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.s, vl16
+; CHECK_BASE_GE_2048-NEXT: mov z1.s, #32767 // =0x7fff
+; CHECK_BASE_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BASE_GE_2048-NEXT: ptrue p0.s
+; CHECK_BASE_GE_2048-NEXT: movprfx z2, z0
+; CHECK_BASE_GE_2048-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK_BASE_GE_2048-NEXT: lsr z3.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: add z5.s, z0.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK_BASE_GE_2048-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK_BASE_GE_2048-NEXT: lsr z4.s, z2.s, #16
+; CHECK_BASE_GE_2048-NEXT: and z3.s, z3.s, #0x1
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z2.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s
+; CHECK_BASE_GE_2048-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK_BASE_GE_2048-NEXT: and z4.s, z4.s, #0x1
+; CHECK_BASE_GE_2048-NEXT: add z3.s, z3.s, z5.s
+; CHECK_BASE_GE_2048-NEXT: add z1.s, z4.s, z1.s
+; CHECK_BASE_GE_2048-NEXT: sel z0.s, p1, z0.s, z3.s
+; CHECK_BASE_GE_2048-NEXT: mov z1.s, p0/m, z2.s
+; CHECK_BASE_GE_2048-NEXT: lsr z0.s, z0.s, #16
+; CHECK_BASE_GE_2048-NEXT: lsr z1.s, z1.s, #16
+; CHECK_BASE_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BASE_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BASE_GE_2048-NEXT: stp q0, q1, [x1]
+; CHECK_BASE_GE_2048-NEXT: ret
+;
+; CHECK_BF16_GE_256-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_256: // %bb.0:
+; CHECK_BF16_GE_256-NEXT: ptrue p0.s, vl8
+; CHECK_BF16_GE_256-NEXT: mov x8, #8 // =0x8
+; CHECK_BF16_GE_256-NEXT: ptrue p1.s
+; CHECK_BF16_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK_BF16_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
+; CHECK_BF16_GE_256-NEXT: bfcvt z0.h, p1/m, z0.s
+; CHECK_BF16_GE_256-NEXT: bfcvt z1.h, p1/m, z1.s
+; CHECK_BF16_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_256-NEXT: stp q1, q0, [x1]
+; CHECK_BF16_GE_256-NEXT: ret
+;
+; CHECK_BF16_GE_512-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_512: // %bb.0:
+; CHECK_BF16_GE_512-NEXT: ptrue p0.s, vl16
+; CHECK_BF16_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BF16_GE_512-NEXT: ptrue p0.s
+; CHECK_BF16_GE_512-NEXT: movprfx z1, z0
+; CHECK_BF16_GE_512-NEXT: ext z1.b, z1.b, z0.b, #32
+; CHECK_BF16_GE_512-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_512-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK_BF16_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_512-NEXT: stp q0, q1, [x1]
+; CHECK_BF16_GE_512-NEXT: ret
+;
+; CHECK_BF16_GE_2048-LABEL: fcvt_v16f32_v16bf16:
+; CHECK_BF16_GE_2048: // %bb.0:
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.s, vl16
+; CHECK_BF16_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK_BF16_GE_2048-NEXT: ptrue p0.s
+; CHECK_BF16_GE_2048-NEXT: movprfx z1, z0
+; CHECK_BF16_GE_2048-NEXT: ext z1.b, z1.b, z0.b, #32
+; CHECK_BF16_GE_2048-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK_BF16_GE_2048-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK_BF16_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK_BF16_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK_BF16_GE_2048-NEXT: stp q0, q1, [x1]
+; CHECK_BF16_GE_2048-NEXT: ret
+ %op1 = load <16 x float>, ptr %a
+ %res = fptrunc <16 x float> %op1 to <16 x bfloat>
+ store <16 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v32f32_v32bf16(ptr %a, ptr %b) vscale_range(8,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v32f32_v32bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl32
+; CHECK-BASE-NEXT: mov z3.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BASE-NEXT: movprfx z2, z0
+; CHECK-BASE-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK-BASE-NEXT: lsr z4.s, z0.s, #16
+; CHECK-BASE-NEXT: add z16.s, z0.s, z3.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: movprfx z7, z1
+; CHECK-BASE-NEXT: ext z7.b, z7.b, z1.b, #32
+; CHECK-BASE-NEXT: lsr z5.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z6.s, z1.s, #16
+; CHECK-BASE-NEXT: add z17.s, z2.s, z3.s
+; CHECK-BASE-NEXT: add z18.s, z1.s, z3.s
+; CHECK-BASE-NEXT: and z4.s, z4.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-BASE-NEXT: orr z2.s, z2.s, #0x400000
+; CHECK-BASE-NEXT: lsr z19.s, z7.s, #16
+; CHECK-BASE-NEXT: and z5.s, z5.s, #0x1
+; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1
+; CHECK-BASE-NEXT: add z3.s, z7.s, z3.s
+; CHECK-BASE-NEXT: add z4.s, z4.s, z16.s
+; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: and z19.s, z19.s, #0x1
+; CHECK-BASE-NEXT: add z5.s, z5.s, z17.s
+; CHECK-BASE-NEXT: add z6.s, z6.s, z18.s
+; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s
+; CHECK-BASE-NEXT: orr z7.s, z7.s, #0x400000
+; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z4.s
+; CHECK-BASE-NEXT: add z3.s, z19.s, z3.s
+; CHECK-BASE-NEXT: sel z2.s, p2, z2.s, z5.s
+; CHECK-BASE-NEXT: sel z1.s, p3, z1.s, z6.s
+; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16
+; CHECK-BASE-NEXT: mov z3.s, p0/m, z7.s
+; CHECK-BASE-NEXT: lsr z2.s, z2.s, #16
+; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16
+; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16
+; CHECK-BASE-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BASE-NEXT: stp q0, q2, [x1]
+; CHECK-BASE-NEXT: stp q1, q3, [x1, #32]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-BF16-LABEL: fcvt_v32f32_v32bf16:
+; CHECK-BF16: // %bb.0:
+; CHECK-BF16-NEXT: ptrue p0.s, vl32
+; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BF16-NEXT: ptrue p0.s
+; CHECK-BF16-NEXT: movprfx z1, z0
+; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #64
+; CHECK-BF16-NEXT: movprfx z2, z0
+; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #32
+; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s
+; CHECK-BF16-NEXT: movprfx z3, z1
+; CHECK-BF16-NEXT: ext z3.b, z3.b, z1.b, #32
+; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s
+; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s
+; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h
+; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s
+; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h
+; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h
+; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h
+; CHECK-BF16-NEXT: stp q0, q2, [x1]
+; CHECK-BF16-NEXT: stp q1, q3, [x1, #32]
+; CHECK-BF16-NEXT: ret
+ %op1 = load <32 x float>, ptr %a
+ %res = fptrunc <32 x float> %op1 to <32 x bfloat>
+ store <32 x bfloat> %res, ptr %b
+ ret void
+}
+
+define void @fcvt_v64f32_v64bf16(ptr %a, ptr %b) vscale_range(16,0) #0 {
+; CHECK-BASE-LABEL: fcvt_v64f32_v64bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ptrue p0.s, vl64
+; CHECK-BASE-NEXT: mov z2.s, #32767 // =0x7fff
+; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0]
+; CHECK-BASE-NEXT: ptrue p0.s
+; CHECK-BASE-NEXT: movprfx z1, z0
+; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #128
+; CHECK-BASE-NEXT: movprfx z3, z0
+; CHECK-BASE-NEXT: ext z3.b, z3.b, z0.b, #32
+; CHECK-BASE-NEXT: movprfx z4, z0
+; CHECK-BASE-NEXT: ext z4.b, z4.b, z0.b, #64
+; CHECK-BASE-NEXT: lsr z16.s, z0.s, #16
+; CHECK-BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s
+; CHECK-BASE-NEXT: lsr z22.s, z1.s, #16
+; CHECK-BASE-NEXT: movprfx z5, z1
+; CHECK-BASE-NEXT: ext z5.b, z5.b, z1.b, #32
+; CHECK-BASE-NEXT: lsr z6.s, z3.s, #16
+; CHECK-BASE-NEXT: lsr z7.s, z4.s, #16
+; CHECK-BASE-NEXT: movprfx z17, z1
+; CHECK-BASE-NEXT: ext z17.b, z17.b, z1.b, #64
+; CHECK-BASE-NEXT: add z23.s, z1.s, z2.s
+; CHECK-BASE-NEXT: movprfx z18, z4
+; CHECK-BASE-NEXT: ext z18.b, z18.b, z4.b, #32
+; CHECK-BASE-NEXT: add z19.s, z3.s, z2.s
+; CHECK-BASE-NEXT: add z21.s, z4.s, z2.s
+; CHECK-BASE-NEXT: and z22.s, z22.s, #0x1
+; CHECK-BASE-NEXT: lsr z20.s, z5.s, #16
+; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1
+; CHECK-BASE-NEXT: and z7.s, z7.s, #0x1
+; CHECK-BASE-NEXT: movprfx z24, z17
+; CHECK-BASE-NEXT: ext z24.b, z24.b, z17.b, #32
+; CHECK-BASE-NEXT: and z16.s, z16.s, #0x1
+; CHECK-BASE-NEXT: add z26.s, z5.s, z2.s
+; CHECK-BASE-NEXT: lsr z25.s, z18.s, #16
+; CHECK-BASE-NEXT: add z22.s, z22.s, z23.s
+; CHECK-BASE-NEXT: add z23.s, z0.s, z2.s
+; CHECK-BASE-NEXT: and z20.s, z20.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s
+; CHECK-BASE-NEXT: add z6.s, z6.s, z19.s
+; CHECK-BASE-NEXT: add z7.s, z7.s, z21.s
+; CHECK-BASE-NEXT: lsr z21.s, z17.s, #16
+; CHECK-BASE-NEXT: orr z3.s, z3.s, #0x400000
+; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000
+; CHECK-BASE-NEXT: add z16.s, z16.s, z23.s
+; CHECK-BASE-NEXT: lsr z23.s, z24.s, #16
+; CHECK-BASE-NEXT: add z20.s, z20.s, z26.s
+; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s
+; CHECK-BASE-NEXT: orr z5.s, z5.s, #0x400000
+; CHECK-BASE-NEXT: add z19.s, z18.s, z2.s
+; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s
+; CHECK-BASE-NEXT: add z26.s, z17.s, z2.s
+; CHECK-BASE-NEXT: and z25.s, z25.s, #0x1
+; CHECK-BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-BASE-NEXT: and z21.s, z21.s, #0x1
+; CHECK-BASE-NEXT: orr z4.s, z4.s, #0x400000
+; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000
+; CHECK-BASE-NEXT: sel z3.s, p2, z3.s, z6.s
+; CHECK-BASE-NEXT: sel z0.s, p5, z0.s, z16.s
+; CHECK-BASE-NEXT: and z23.s, z23.s, #0x1
+; CHECK-BASE-NEX...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/210264
More information about the llvm-branch-commits
mailing list