[llvm] [AArch64] Lower bfloat FADD/SUB to BFMLAL top/bottom instructions (PR #174814)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 26 09:41:22 PST 2026
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/174814
>From 3508eea71762b01327af7c1726bfeb7e85a36187 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Fri, 9 Jan 2026 15:55:07 +0000
Subject: [PATCH 1/2] [AArch64] Lower bfloat FADD/SUB to BFMLAL top/bottom
instructions
Assuming all constant/invariant operations are hoisted, this lowering
should improve throughput for Neon and SVE.
Neon: https://godbolt.org/z/eM7qEfExv
SVE: https://godbolt.org/z/8jv19eGf3
This lowering works by extending the even/odd lanes of the LHS to be
used in the accumulators of the BFMLAL instructions. The RHS is then
used as one of the operands to the BFMLAL with the other a constant 1.0
(or -1.0 for FSUB).
---
.../Target/AArch64/AArch64ISelLowering.cpp | 83 ++++--
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 3 +-
.../CodeGen/AArch64/bf16-v8-instructions.ll | 73 +++--
.../AArch64/fixed-length-bf16-arith.ll | 37 +--
llvm/test/CodeGen/AArch64/sve-bf16-arith.ll | 92 ++++---
.../test/CodeGen/AArch64/sve-bf16-combines.ll | 251 +++++++-----------
6 files changed, 284 insertions(+), 255 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 860fd173b9b34..b12c312e20a07 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1853,22 +1853,28 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
!Subtarget->isNonStreamingSVEorSME2Available()) {
for (MVT VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
MVT PromotedVT = VT.changeVectorElementType(MVT::f32);
- setOperationPromotedToType(ISD::FADD, VT, PromotedVT);
setOperationPromotedToType(ISD::FMA, VT, PromotedVT);
setOperationPromotedToType(ISD::FMAXIMUM, VT, PromotedVT);
setOperationPromotedToType(ISD::FMAXNUM, VT, PromotedVT);
setOperationPromotedToType(ISD::FMINIMUM, VT, PromotedVT);
setOperationPromotedToType(ISD::FMINNUM, VT, PromotedVT);
- setOperationPromotedToType(ISD::FSUB, VT, PromotedVT);
- if (VT != MVT::nxv2bf16 && Subtarget->hasBF16())
+ if (VT != MVT::nxv2bf16 && Subtarget->hasBF16()) {
setOperationAction(ISD::FMUL, VT, Custom);
- else
+ setOperationAction(ISD::FADD, VT, Custom);
+ setOperationAction(ISD::FSUB, VT, Custom);
+ } else {
setOperationPromotedToType(ISD::FMUL, VT, PromotedVT);
+ setOperationPromotedToType(ISD::FADD, VT, PromotedVT);
+ setOperationPromotedToType(ISD::FSUB, VT, PromotedVT);
+ }
}
- if (Subtarget->hasBF16() && Subtarget->isNeonAvailable())
+ if (Subtarget->hasBF16() && Subtarget->isNeonAvailable()) {
setOperationAction(ISD::FMUL, MVT::v8bf16, Custom);
+ setOperationAction(ISD::FADD, MVT::v8bf16, Custom);
+ setOperationAction(ISD::FSUB, MVT::v8bf16, Custom);
+ }
}
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
@@ -7816,17 +7822,21 @@ SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
EndOfTrmp);
}
-SDValue AArch64TargetLowering::LowerFMUL(SDValue Op, SelectionDAG &DAG) const {
+SDValue
+AArch64TargetLowering::LowerBFloatArithToBFMLAL(SDValue Op,
+ SelectionDAG &DAG) const {
SDLoc DL(Op);
+ unsigned Opcode = Op.getOpcode();
EVT VT = Op.getValueType();
- if (VT.getScalarType() != MVT::bf16 ||
+ if ((Opcode != ISD::FADD && Opcode != ISD::FSUB && Opcode != ISD::FMUL) ||
+ VT.getScalarType() != MVT::bf16 ||
(Subtarget->hasSVEB16B16() &&
Subtarget->isNonStreamingSVEorSME2Available()))
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
+ return SDValue();
- assert(Subtarget->hasBF16() && "Expected +bf16 for custom FMUL lowering");
+ assert(Subtarget->hasBF16() && "Expected +bf16 for custom FMUL/ADD lowering");
assert((VT == MVT::nxv4bf16 || VT == MVT::nxv8bf16 || VT == MVT::v8bf16) &&
- "Unexpected FMUL VT");
+ "Unexpected FMUL/ADD VT");
auto MakeGetIntrinsic = [&](Intrinsic::ID IID) {
return [&, IID](EVT VT, auto... Ops) {
@@ -7860,16 +7870,41 @@ SDValue AArch64TargetLowering::LowerFMUL(SDValue Op, SelectionDAG &DAG) const {
: Intrinsic::aarch64_neon_bfmlalt);
EVT AccVT = UseSVEBFMLAL ? MVT::nxv4f32 : MVT::v4f32;
- bool IgnoreZeroSign =
- Op->getFlags().hasNoSignedZeros() || DAG.canIgnoreSignBitOfZero(Op);
- SDValue Zero = DAG.getConstantFP(IgnoreZeroSign ? +0.0F : -0.0F, DL, AccVT);
SDValue Pg = getPredicateForVector(DAG, DL, AccVT);
-
- // Lower bf16 FMUL as a pair (VT == [nx]v8bf16) of BFMLAL top/bottom
- // instructions. These result in two f32 vectors, which can be converted back
- // to bf16 with FCVT and FCVTNT.
- SDValue LHS = Op.getOperand(0);
- SDValue RHS = Op.getOperand(1);
+ SDValue BottomAcc, TopAcc, LHS, RHS;
+
+ if (Opcode == ISD::FMUL) {
+ bool IgnoreZeroSign =
+ Op->getFlags().hasNoSignedZeros() || DAG.canIgnoreSignBitOfZero(Op);
+ // Set both accumulators to zero.
+ BottomAcc = TopAcc =
+ DAG.getConstantFP(IgnoreZeroSign ? +0.0F : -0.0F, DL, AccVT);
+ // Lower bf16 FMUL as a pair (VT == [nx]v8bf16) of BFMLAL top/bottom
+ // instructions. These result in two f32 vectors, which can be converted
+ // back to bf16 with FCVT and FCVTNT.
+ LHS = Op.getOperand(0);
+ RHS = Op.getOperand(1);
+ } else if (Opcode == ISD::FADD || Opcode == ISD::FSUB) {
+ // Lower FADD/SUB by extending the LHS to be used as the accumulator, and
+ // multiplying the RHS by 1.0F (or -1.0F for FSUB).
+ if (VT.getVectorMinNumElements() > 4) {
+ SDValue Zero = DAG.getConstantFP(+0.0, DL, VT);
+ // Note: This extends the even/odd lanes to f32. TRN1/2 is used as the
+ // zero is likely cheap/hoisted and can be used to extend the odd lanes.
+ BottomAcc = DAG.getNode(
+ AArch64ISD::NVCAST, DL, AccVT,
+ DAG.getNode(AArch64ISD::TRN1, DL, VT, Zero, Op.getOperand(0)));
+ TopAcc = DAG.getNode(
+ AArch64ISD::NVCAST, DL, AccVT,
+ DAG.getNode(AArch64ISD::TRN2, DL, VT, Zero, Op.getOperand(0)));
+ } else {
+ BottomAcc = DAG.getNode(ISD::FP_EXTEND, DL, AccVT, Op.getOperand(0));
+ }
+ LHS = Op.getOperand(1);
+ RHS = DAG.getConstantFP(Opcode == ISD::FSUB ? -1.0F : 1.0F, DL, VT);
+ } else {
+ llvm_unreachable("Unexpected operation");
+ }
// All SVE intrinsics expect to operate on full bf16 vector types.
if (UseSVEBFMLAL) {
@@ -7877,14 +7912,15 @@ SDValue AArch64TargetLowering::LowerFMUL(SDValue Op, SelectionDAG &DAG) const {
RHS = Reinterpret(RHS, MVT::nxv8bf16);
}
- SDValue BottomF32 = Reinterpret(BFMLALB(AccVT, Zero, LHS, RHS), MVT::nxv4f32);
+ SDValue BottomF32 =
+ Reinterpret(BFMLALB(AccVT, BottomAcc, LHS, RHS), MVT::nxv4f32);
SDValue BottomBF16 =
FCVT(MVT::nxv8bf16, DAG.getPOISON(MVT::nxv8bf16), Pg, BottomF32);
// Note: nxv4bf16 only uses even lanes.
if (VT == MVT::nxv4bf16)
return Reinterpret(BottomBF16, VT);
- SDValue TopF32 = Reinterpret(BFMLALT(AccVT, Zero, LHS, RHS), MVT::nxv4f32);
+ SDValue TopF32 = Reinterpret(BFMLALT(AccVT, TopAcc, LHS, RHS), MVT::nxv4f32);
SDValue TopBF16 = FCVTNT(MVT::nxv8bf16, BottomBF16, Pg, TopF32);
return Reinterpret(TopBF16, VT);
}
@@ -7934,6 +7970,9 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
LLVM_DEBUG(dbgs() << "Custom lowering: ");
LLVM_DEBUG(Op.dump());
+ if (SDValue Result = LowerBFloatArithToBFMLAL(Op, DAG))
+ return Result;
+
switch (Op.getOpcode()) {
default:
llvm_unreachable("unimplemented operand");
@@ -8003,7 +8042,7 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::FSUB:
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
case ISD::FMUL:
- return LowerFMUL(Op, DAG);
+ return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
case ISD::FMA:
return LowerFMA(Op, DAG);
case ISD::FDIV:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 8f07608faa710..37c9c60f9e199 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -620,7 +620,6 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerStore128(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
- SDValue LowerFMUL(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerFMA(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
@@ -634,6 +633,8 @@ class AArch64TargetLowering : public TargetLowering {
SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBFloatArithToBFMLAL(SDValue Op, SelectionDAG &DAG) const;
+
bool
isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;
diff --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index e3c0d97c08f54..7176c6076fa8d 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -30,17 +30,31 @@ define <8 x bfloat> @add_h(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h
; CHECK-CVT-NEXT: ret
;
-; CHECK-BF16-LABEL: add_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fadd v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-NOSVE-BF16-LABEL: add_h:
+; CHECK-NOSVE-BF16: // %bb.0: // %entry
+; CHECK-NOSVE-BF16-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-NOSVE-BF16-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-NOSVE-BF16-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NOSVE-BF16-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NOSVE-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
+; CHECK-NOSVE-BF16-NEXT: fadd v1.4s, v0.4s, v1.4s
+; CHECK-NOSVE-BF16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-NOSVE-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-NOSVE-BF16-NEXT: ret
+;
+; CHECK-SVE-BF16-LABEL: add_h:
+; CHECK-SVE-BF16: // %bb.0: // %entry
+; CHECK-SVE-BF16-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SVE-BF16-NEXT: mov z4.h, #16256 // =0x3f80
+; CHECK-SVE-BF16-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-BF16-NEXT: trn1 v3.8h, v2.8h, v0.8h
+; CHECK-SVE-BF16-NEXT: trn2 v2.8h, v2.8h, v0.8h
+; CHECK-SVE-BF16-NEXT: bfmlalb v3.4s, v1.8h, v4.8h
+; CHECK-SVE-BF16-NEXT: bfmlalt v2.4s, v1.8h, v4.8h
+; CHECK-SVE-BF16-NEXT: bfcvt z0.h, p0/m, z3.s
+; CHECK-SVE-BF16-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; CHECK-SVE-BF16-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-BF16-NEXT: ret
entry:
%0 = fadd <8 x bfloat> %a, %b
ret <8 x bfloat> %0
@@ -74,17 +88,32 @@ define <8 x bfloat> @sub_h(<8 x bfloat> %a, <8 x bfloat> %b) {
; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h
; CHECK-CVT-NEXT: ret
;
-; CHECK-BF16-LABEL: sub_h:
-; CHECK-BF16: // %bb.0: // %entry
-; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16
-; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16
-; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16
-; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16
-; CHECK-BF16-NEXT: fsub v2.4s, v3.4s, v2.4s
-; CHECK-BF16-NEXT: fsub v1.4s, v0.4s, v1.4s
-; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s
-; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
-; CHECK-BF16-NEXT: ret
+; CHECK-NOSVE-BF16-LABEL: sub_h:
+; CHECK-NOSVE-BF16: // %bb.0: // %entry
+; CHECK-NOSVE-BF16-NEXT: shll v2.4s, v1.4h, #16
+; CHECK-NOSVE-BF16-NEXT: shll v3.4s, v0.4h, #16
+; CHECK-NOSVE-BF16-NEXT: shll2 v1.4s, v1.8h, #16
+; CHECK-NOSVE-BF16-NEXT: shll2 v0.4s, v0.8h, #16
+; CHECK-NOSVE-BF16-NEXT: fsub v2.4s, v3.4s, v2.4s
+; CHECK-NOSVE-BF16-NEXT: fsub v1.4s, v0.4s, v1.4s
+; CHECK-NOSVE-BF16-NEXT: bfcvtn v0.4h, v2.4s
+; CHECK-NOSVE-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
+; CHECK-NOSVE-BF16-NEXT: ret
+;
+; CHECK-SVE-BF16-LABEL: sub_h:
+; CHECK-SVE-BF16: // %bb.0: // %entry
+; CHECK-SVE-BF16-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SVE-BF16-NEXT: mov w8, #49024 // =0xbf80
+; CHECK-SVE-BF16-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-BF16-NEXT: dup v4.8h, w8
+; CHECK-SVE-BF16-NEXT: trn1 v3.8h, v2.8h, v0.8h
+; CHECK-SVE-BF16-NEXT: trn2 v2.8h, v2.8h, v0.8h
+; CHECK-SVE-BF16-NEXT: bfmlalb v3.4s, v1.8h, v4.8h
+; CHECK-SVE-BF16-NEXT: bfmlalt v2.4s, v1.8h, v4.8h
+; CHECK-SVE-BF16-NEXT: bfcvt z0.h, p0/m, z3.s
+; CHECK-SVE-BF16-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; CHECK-SVE-BF16-NEXT: // kill: def $q0 killed $q0 killed $z0
+; CHECK-SVE-BF16-NEXT: ret
entry:
%0 = fsub <8 x bfloat> %a, %b
ret <8 x bfloat> %0
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-bf16-arith.ll b/llvm/test/CodeGen/AArch64/fixed-length-bf16-arith.ll
index 9147e56dde037..0c04dce759f04 100644
--- a/llvm/test/CodeGen/AArch64/fixed-length-bf16-arith.ll
+++ b/llvm/test/CodeGen/AArch64/fixed-length-bf16-arith.ll
@@ -54,14 +54,16 @@ define <4 x bfloat> @fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
define <8 x bfloat> @fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; NOB16B16-LABEL: fadd_v8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: shll v2.4s, v1.4h, #16
-; NOB16B16-NEXT: shll v3.4s, v0.4h, #16
-; NOB16B16-NEXT: shll2 v1.4s, v1.8h, #16
-; NOB16B16-NEXT: shll2 v0.4s, v0.8h, #16
-; NOB16B16-NEXT: fadd v2.4s, v3.4s, v2.4s
-; NOB16B16-NEXT: fadd v1.4s, v0.4s, v1.4s
-; NOB16B16-NEXT: bfcvtn v0.4h, v2.4s
-; NOB16B16-NEXT: bfcvtn2 v0.8h, v1.4s
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
+; NOB16B16-NEXT: mov z4.h, #16256 // =0x3f80
+; NOB16B16-NEXT: ptrue p0.s, vl4
+; NOB16B16-NEXT: trn1 v3.8h, v2.8h, v0.8h
+; NOB16B16-NEXT: trn2 v2.8h, v2.8h, v0.8h
+; NOB16B16-NEXT: bfmlalb v3.4s, v1.8h, v4.8h
+; NOB16B16-NEXT: bfmlalt v2.4s, v1.8h, v4.8h
+; NOB16B16-NEXT: bfcvt z0.h, p0/m, z3.s
+; NOB16B16-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-NEXT: // kill: def $q0 killed $q0 killed $z0
; NOB16B16-NEXT: ret
;
; B16B16-LABEL: fadd_v8bf16:
@@ -943,14 +945,17 @@ define <4 x bfloat> @fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
define <8 x bfloat> @fsub_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; NOB16B16-LABEL: fsub_v8bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: shll v2.4s, v1.4h, #16
-; NOB16B16-NEXT: shll v3.4s, v0.4h, #16
-; NOB16B16-NEXT: shll2 v1.4s, v1.8h, #16
-; NOB16B16-NEXT: shll2 v0.4s, v0.8h, #16
-; NOB16B16-NEXT: fsub v2.4s, v3.4s, v2.4s
-; NOB16B16-NEXT: fsub v1.4s, v0.4s, v1.4s
-; NOB16B16-NEXT: bfcvtn v0.4h, v2.4s
-; NOB16B16-NEXT: bfcvtn2 v0.8h, v1.4s
+; NOB16B16-NEXT: movi v2.2d, #0000000000000000
+; NOB16B16-NEXT: mov w8, #49024 // =0xbf80
+; NOB16B16-NEXT: ptrue p0.s, vl4
+; NOB16B16-NEXT: dup v4.8h, w8
+; NOB16B16-NEXT: trn1 v3.8h, v2.8h, v0.8h
+; NOB16B16-NEXT: trn2 v2.8h, v2.8h, v0.8h
+; NOB16B16-NEXT: bfmlalb v3.4s, v1.8h, v4.8h
+; NOB16B16-NEXT: bfmlalt v2.4s, v1.8h, v4.8h
+; NOB16B16-NEXT: bfcvt z0.h, p0/m, z3.s
+; NOB16B16-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-NEXT: // kill: def $q0 killed $q0 killed $z0
; NOB16B16-NEXT: ret
;
; B16B16-LABEL: fsub_v8bf16:
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
index 6917ac12999bf..89d2f5485fd38 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
@@ -66,10 +66,10 @@ define <vscale x 2 x bfloat> @fadd_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x
define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fadd_nxv4bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
+; NOB16B16-NEXT: fmov z2.h, #1.87500000
; NOB16B16-NEXT: lsl z0.s, z0.s, #16
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s
+; NOB16B16-NEXT: bfmlalb z0.s, z1.h, z2.h
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: ret
;
@@ -83,28 +83,36 @@ define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
}
define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
-; NOB16B16-LABEL: fadd_nxv8bf16:
-; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
-; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fadd z2.s, z3.s, z2.s
-; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
-; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
-; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
-; NOB16B16-NEXT: ret
+; NOB16B16-NONSTREAMING-LABEL: fadd_nxv8bf16:
+; NOB16B16-NONSTREAMING: // %bb.0:
+; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000
+; NOB16B16-NONSTREAMING-NEXT: fmov z3.h, #1.87500000
+; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s
+; NOB16B16-NONSTREAMING-NEXT: trn1 z4.h, z2.h, z0.h
+; NOB16B16-NONSTREAMING-NEXT: trn2 z2.h, z2.h, z0.h
+; NOB16B16-NONSTREAMING-NEXT: bfmlalb z4.s, z1.h, z3.h
+; NOB16B16-NONSTREAMING-NEXT: bfmlalt z2.s, z1.h, z3.h
+; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z4.s
+; NOB16B16-NONSTREAMING-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-NONSTREAMING-NEXT: ret
;
; B16B16-LABEL: fadd_nxv8bf16:
; B16B16: // %bb.0:
; B16B16-NEXT: bfadd z0.h, z0.h, z1.h
; B16B16-NEXT: ret
+;
+; NOB16B16-STREAMING-LABEL: fadd_nxv8bf16:
+; NOB16B16-STREAMING: // %bb.0:
+; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0
+; NOB16B16-STREAMING-NEXT: fmov z3.h, #1.87500000
+; NOB16B16-STREAMING-NEXT: ptrue p0.s
+; NOB16B16-STREAMING-NEXT: trn1 z4.h, z2.h, z0.h
+; NOB16B16-STREAMING-NEXT: trn2 z2.h, z2.h, z0.h
+; NOB16B16-STREAMING-NEXT: bfmlalb z4.s, z1.h, z3.h
+; NOB16B16-STREAMING-NEXT: bfmlalt z2.s, z1.h, z3.h
+; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z4.s
+; NOB16B16-STREAMING-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-STREAMING-NEXT: ret
%res = fadd <vscale x 8 x bfloat> %a, %b
ret <vscale x 8 x bfloat> %res
}
@@ -717,10 +725,10 @@ define <vscale x 2 x bfloat> @fsub_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x
define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
; NOB16B16-LABEL: fsub_nxv4bf16:
; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
+; NOB16B16-NEXT: fmov z2.h, #-1.87500000
; NOB16B16-NEXT: lsl z0.s, z0.s, #16
; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s
+; NOB16B16-NEXT: bfmlalb z0.s, z1.h, z2.h
; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
; NOB16B16-NEXT: ret
;
@@ -734,28 +742,36 @@ define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x
}
define <vscale x 8 x bfloat> @fsub_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
-; NOB16B16-LABEL: fsub_nxv8bf16:
-; NOB16B16: // %bb.0:
-; NOB16B16-NEXT: uunpkhi z2.s, z1.h
-; NOB16B16-NEXT: uunpkhi z3.s, z0.h
-; NOB16B16-NEXT: uunpklo z1.s, z1.h
-; NOB16B16-NEXT: uunpklo z0.s, z0.h
-; NOB16B16-NEXT: ptrue p0.s
-; NOB16B16-NEXT: lsl z2.s, z2.s, #16
-; NOB16B16-NEXT: lsl z3.s, z3.s, #16
-; NOB16B16-NEXT: lsl z1.s, z1.s, #16
-; NOB16B16-NEXT: lsl z0.s, z0.s, #16
-; NOB16B16-NEXT: fsub z2.s, z3.s, z2.s
-; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s
-; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s
-; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s
-; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h
-; NOB16B16-NEXT: ret
+; NOB16B16-NONSTREAMING-LABEL: fsub_nxv8bf16:
+; NOB16B16-NONSTREAMING: // %bb.0:
+; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000
+; NOB16B16-NONSTREAMING-NEXT: fmov z3.h, #-1.87500000
+; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s
+; NOB16B16-NONSTREAMING-NEXT: trn1 z4.h, z2.h, z0.h
+; NOB16B16-NONSTREAMING-NEXT: trn2 z2.h, z2.h, z0.h
+; NOB16B16-NONSTREAMING-NEXT: bfmlalb z4.s, z1.h, z3.h
+; NOB16B16-NONSTREAMING-NEXT: bfmlalt z2.s, z1.h, z3.h
+; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z4.s
+; NOB16B16-NONSTREAMING-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-NONSTREAMING-NEXT: ret
;
; B16B16-LABEL: fsub_nxv8bf16:
; B16B16: // %bb.0:
; B16B16-NEXT: bfsub z0.h, z0.h, z1.h
; B16B16-NEXT: ret
+;
+; NOB16B16-STREAMING-LABEL: fsub_nxv8bf16:
+; NOB16B16-STREAMING: // %bb.0:
+; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0
+; NOB16B16-STREAMING-NEXT: fmov z3.h, #-1.87500000
+; NOB16B16-STREAMING-NEXT: ptrue p0.s
+; NOB16B16-STREAMING-NEXT: trn1 z4.h, z2.h, z0.h
+; NOB16B16-STREAMING-NEXT: trn2 z2.h, z2.h, z0.h
+; NOB16B16-STREAMING-NEXT: bfmlalb z4.s, z1.h, z3.h
+; NOB16B16-STREAMING-NEXT: bfmlalt z2.s, z1.h, z3.h
+; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z4.s
+; NOB16B16-STREAMING-NEXT: bfcvtnt z0.h, p0/m, z2.s
+; NOB16B16-STREAMING-NEXT: ret
%res = fsub <vscale x 8 x bfloat> %a, %b
ret <vscale x 8 x bfloat> %res
}
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll b/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
index a8049806a679b..64fea57ffe66a 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-combines.ll
@@ -292,20 +292,15 @@ define <vscale x 2 x bfloat> @fmls_sel_nxv2bf16(<vscale x 2 x i1> %pred, <vscale
define <vscale x 8 x bfloat> @fadd_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fadd_sel_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -321,20 +316,15 @@ define <vscale x 8 x bfloat> @fadd_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscal
define <vscale x 8 x bfloat> @fsub_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fsub_sel_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
+; SVE-NEXT: fmov z3.h, #-1.87500000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -350,20 +340,15 @@ define <vscale x 8 x bfloat> @fsub_sel_nxv8bf16(<vscale x 8 x bfloat> %a, <vscal
define <vscale x 8 x bfloat> @fadd_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fadd_sel_negzero_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -380,20 +365,15 @@ define <vscale x 8 x bfloat> @fadd_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a
define <vscale x 8 x bfloat> @fsub_sel_negzero_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x i1> %mask) {
; SVE-LABEL: fsub_sel_negzero_nxv8bf16:
; SVE: // %bb.0:
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: uunpklo z4.s, z0.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
+; SVE-NEXT: fmov z3.h, #-1.87500000
; SVE-NEXT: ptrue p1.s
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -417,21 +397,15 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
; SVE-NEXT: sel z1.h, p0, z1.h, z2.h
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z0.s, z0.s, z1.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z0.h, p1/m, z0.s
-; SVE-NEXT: uzp1 z0.h, z0.h, z1.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z0.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z0.h, p1/m, z2.s
; SVE-NEXT: ret
;
; SVE-B16B16-LABEL: fadd_sel_fmul_nxv8bf16:
@@ -455,21 +429,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_nxv8bf16(<vscale x 8 x bfloat> %a, <
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #-1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -491,21 +460,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -527,21 +491,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_nsz_nxv8bf16(<vscale x 8 x bfloat> %
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #-1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -563,21 +522,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloa
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -602,21 +556,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nxv8bf16(<vscale x 8 x bfloa
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
; SVE-NEXT: dupm z2.h, #0x8000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
-; SVE-NEXT: uunpklo z0.s, z0.h
+; SVE-NEXT: movi v3.2d, #0000000000000000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: lsl z0.s, z0.s, #16
+; SVE-NEXT: trn1 z4.h, z3.h, z0.h
+; SVE-NEXT: trn2 z3.h, z3.h, z0.h
; SVE-NEXT: sel z1.h, p0, z1.h, z2.h
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z0.s, z0.s, z1.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z0.h, p1/m, z0.s
-; SVE-NEXT: uzp1 z0.h, z0.h, z1.h
+; SVE-NEXT: fmov z2.h, #-1.87500000
+; SVE-NEXT: bfmlalb z4.s, z1.h, z2.h
+; SVE-NEXT: bfmlalt z3.s, z1.h, z2.h
+; SVE-NEXT: bfcvt z0.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z0.h, p1/m, z3.s
; SVE-NEXT: ret
;
; SVE-B16B16-LABEL: fsub_sel_fmul_negzero_nxv8bf16:
@@ -641,21 +590,16 @@ define <vscale x 8 x bfloat> @fadd_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x b
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fadd z2.s, z3.s, z2.s
-; SVE-NEXT: fadd z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
@@ -678,21 +622,16 @@ define <vscale x 8 x bfloat> @fsub_sel_fmul_negzero_nsz_nxv8bf16(<vscale x 8 x b
; SVE-NEXT: ptrue p1.s
; SVE-NEXT: bfmlalb z3.s, z1.h, z2.h
; SVE-NEXT: bfmlalt z4.s, z1.h, z2.h
+; SVE-NEXT: movi v2.2d, #0000000000000000
; SVE-NEXT: bfcvt z1.h, p1/m, z3.s
-; SVE-NEXT: uunpkhi z3.s, z0.h
+; SVE-NEXT: fmov z3.h, #-1.87500000
; SVE-NEXT: bfcvtnt z1.h, p1/m, z4.s
-; SVE-NEXT: uunpklo z4.s, z0.h
-; SVE-NEXT: lsl z3.s, z3.s, #16
-; SVE-NEXT: uunpkhi z2.s, z1.h
-; SVE-NEXT: uunpklo z1.s, z1.h
-; SVE-NEXT: lsl z4.s, z4.s, #16
-; SVE-NEXT: lsl z2.s, z2.s, #16
-; SVE-NEXT: lsl z1.s, z1.s, #16
-; SVE-NEXT: fsub z2.s, z3.s, z2.s
-; SVE-NEXT: fsub z1.s, z4.s, z1.s
-; SVE-NEXT: bfcvt z2.h, p1/m, z2.s
-; SVE-NEXT: bfcvt z1.h, p1/m, z1.s
-; SVE-NEXT: uzp1 z1.h, z1.h, z2.h
+; SVE-NEXT: trn1 z4.h, z2.h, z0.h
+; SVE-NEXT: trn2 z2.h, z2.h, z0.h
+; SVE-NEXT: bfmlalb z4.s, z1.h, z3.h
+; SVE-NEXT: bfmlalt z2.s, z1.h, z3.h
+; SVE-NEXT: bfcvt z1.h, p1/m, z4.s
+; SVE-NEXT: bfcvtnt z1.h, p1/m, z2.s
; SVE-NEXT: mov z0.h, p0/m, z1.h
; SVE-NEXT: ret
;
>From 4a8f57ca28b71f3967ded84692bea5458c0e577b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 26 Jan 2026 17:39:23 +0000
Subject: [PATCH 2/2] Fixup
---
.../Target/AArch64/AArch64ISelLowering.cpp | 22 +++++++++++++------
1 file changed, 15 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b12c312e20a07..5d7dd04507749 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1860,6 +1860,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationPromotedToType(ISD::FMINNUM, VT, PromotedVT);
if (VT != MVT::nxv2bf16 && Subtarget->hasBF16()) {
+ // These operations can be lowered to BFMLAT/B. Note: nxv2bf16 is not
+ // supported to avoid performing fp operations on inactive lanes (as
+ // BFMLAT/B is unpredicated).
setOperationAction(ISD::FMUL, VT, Custom);
setOperationAction(ISD::FADD, VT, Custom);
setOperationAction(ISD::FSUB, VT, Custom);
@@ -7827,16 +7830,18 @@ AArch64TargetLowering::LowerBFloatArithToBFMLAL(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
unsigned Opcode = Op.getOpcode();
+ assert((Opcode == ISD::FADD || Opcode == ISD::FSUB || Opcode == ISD::FMUL) &&
+ "Unexpected operation");
+
EVT VT = Op.getValueType();
- if ((Opcode != ISD::FADD && Opcode != ISD::FSUB && Opcode != ISD::FMUL) ||
- VT.getScalarType() != MVT::bf16 ||
+ if (VT.getScalarType() != MVT::bf16 ||
(Subtarget->hasSVEB16B16() &&
Subtarget->isNonStreamingSVEorSME2Available()))
return SDValue();
assert(Subtarget->hasBF16() && "Expected +bf16 for custom FMUL/ADD lowering");
assert((VT == MVT::nxv4bf16 || VT == MVT::nxv8bf16 || VT == MVT::v8bf16) &&
- "Unexpected FMUL/ADD VT");
+ "Unexpected VT");
auto MakeGetIntrinsic = [&](Intrinsic::ID IID) {
return [&, IID](EVT VT, auto... Ops) {
@@ -7887,7 +7892,7 @@ AArch64TargetLowering::LowerBFloatArithToBFMLAL(SDValue Op,
} else if (Opcode == ISD::FADD || Opcode == ISD::FSUB) {
// Lower FADD/SUB by extending the LHS to be used as the accumulator, and
// multiplying the RHS by 1.0F (or -1.0F for FSUB).
- if (VT.getVectorMinNumElements() > 4) {
+ if (VT != MVT::nxv4bf16) {
SDValue Zero = DAG.getConstantFP(+0.0, DL, VT);
// Note: This extends the even/odd lanes to f32. TRN1/2 is used as the
// zero is likely cheap/hoisted and can be used to extend the odd lanes.
@@ -7970,9 +7975,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
LLVM_DEBUG(dbgs() << "Custom lowering: ");
LLVM_DEBUG(Op.dump());
- if (SDValue Result = LowerBFloatArithToBFMLAL(Op, DAG))
- return Result;
-
switch (Op.getOpcode()) {
default:
llvm_unreachable("unimplemented operand");
@@ -8038,10 +8040,16 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::UMULO:
return LowerXALUO(Op, DAG);
case ISD::FADD:
+ if (SDValue Result = LowerBFloatArithToBFMLAL(Op, DAG))
+ return Result;
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FADD_PRED);
case ISD::FSUB:
+ if (SDValue Result = LowerBFloatArithToBFMLAL(Op, DAG))
+ return Result;
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FSUB_PRED);
case ISD::FMUL:
+ if (SDValue Result = LowerBFloatArithToBFMLAL(Op, DAG))
+ return Result;
return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMUL_PRED);
case ISD::FMA:
return LowerFMA(Op, DAG);
More information about the llvm-commits
mailing list