[llvm] 0725c48 - [AArch64] Fix widening of half/bfloat partial reductions. (#220261)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 2 09:09:58 PDT 2026
Author: Graham Hunter
Date: 2026-09-02T17:09:52+01:00
New Revision: 0725c489f8df6e23583997738be993c0c21b07d9
URL: https://github.com/llvm/llvm-project/commit/0725c489f8df6e23583997738be993c0c21b07d9
DIFF: https://github.com/llvm/llvm-project/commit/0725c489f8df6e23583997738be993c0c21b07d9.diff
LOG: [AArch64] Fix widening of half/bfloat partial reductions. (#220261)
Includes testing aimed at letting us use partial reductions as the
default for add/fadd reductions when ordering isn't required.
Added:
llvm/test/CodeGen/AArch64/default-partial-reduce.ll
Modified:
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index bc52b8357582c..d6d9617440fb7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -1106,6 +1106,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue WidenVecRes_Unary(SDNode *N);
SDValue WidenVecRes_InregOp(SDNode *N);
SDValue WidenVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo);
+ SDValue WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N);
void ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,
unsigned WidenResNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 0a72bff72dce3..b0480feeb6a6d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5534,6 +5534,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
Res = WidenVecRes_UnaryOpWithTwoResults(N, ResNo);
break;
}
+ case ISD::PARTIAL_REDUCE_FMLA:
+ Res = WidenVecRes_PARTIAL_REDUCE_MLA(N);
+ break;
}
// If Res is null, the sub-method took care of registering the result.
@@ -7594,6 +7597,16 @@ SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
return DAG.getBuildVector(WidenVT, dl, Scalars);
}
+SDValue DAGTypeLegalizer::WidenVecRes_PARTIAL_REDUCE_MLA(SDNode *N) {
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+
+ // Expand, then widen the result.
+ SDValue Expanded = TLI.expandPartialReduceMLA(N, DAG);
+ EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getInsertSubvector(DL, DAG.getPOISON(WideVT), Expanded, 0);
+}
+
//===----------------------------------------------------------------------===//
// Widen Vector Operand
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AArch64/default-partial-reduce.ll b/llvm/test/CodeGen/AArch64/default-partial-reduce.ll
new file mode 100644
index 0000000000000..e47969b6b2d9a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/default-partial-reduce.ll
@@ -0,0 +1,1150 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s | FileCheck %s --check-prefixes=NEON-NOBF16
+; RUN: llc -mattr=+bf16 < %s | FileCheck %s --check-prefixes=NEON-BF16
+; RUN: llc -mattr=+sve2,+bf16,+sve-b16b16, < %s | FileCheck %s --check-prefixes=SVE
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Test partial reductions in place of normal vector operations of the same
+;; size.
+
+;; Int
+define <4 x i8> @pr_4xi8(<4 x i8> %in, <4 x i8> %acc) {
+; NEON-NOBF16-LABEL: pr_4xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: bic v0.4h, #255, lsl #8
+; NEON-NOBF16-NEXT: add v0.4h, v1.4h, v0.4h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: bic v0.4h, #255, lsl #8
+; NEON-BF16-NEXT: add v0.4h, v1.4h, v0.4h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: bic v0.4h, #255, lsl #8
+; SVE-NEXT: add v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %res = call <4 x i8> @llvm.vector.partial.reduce.add(<4 x i8> %acc, <4 x i8> %in)
+ ret <4 x i8> %res
+}
+
+define <8 x i8> @pr_8xi8(<8 x i8> %in, <8 x i8> %acc) {
+; NEON-NOBF16-LABEL: pr_8xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.8b, v1.8b, v0.8b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.8b, v1.8b, v0.8b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.8b, v1.8b, v0.8b
+; SVE-NEXT: ret
+ %res = call <8 x i8> @llvm.vector.partial.reduce.add(<8 x i8> %acc, <8 x i8> %in)
+ ret <8 x i8> %res
+}
+
+define <16 x i8> @pr_16xi8(<16 x i8> %in, <16 x i8> %acc) {
+; NEON-NOBF16-LABEL: pr_16xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.16b, v1.16b, v0.16b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_16xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.16b, v1.16b, v0.16b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_16xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.16b, v1.16b, v0.16b
+; SVE-NEXT: ret
+ %res = call <16 x i8> @llvm.vector.partial.reduce.add(<16 x i8> %acc, <16 x i8> %in)
+ ret <16 x i8> %res
+}
+
+define <32 x i8> @pr_32xi8(<32 x i8> %in, <32 x i8> %acc) {
+; NEON-NOBF16-LABEL: pr_32xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v1.16b, v3.16b, v1.16b
+; NEON-NOBF16-NEXT: add v0.16b, v2.16b, v0.16b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_32xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v1.16b, v3.16b, v1.16b
+; NEON-BF16-NEXT: add v0.16b, v2.16b, v0.16b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_32xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: add v1.16b, v3.16b, v1.16b
+; SVE-NEXT: add v0.16b, v2.16b, v0.16b
+; SVE-NEXT: ret
+ %res = call <32 x i8> @llvm.vector.partial.reduce.add(<32 x i8> %acc, <32 x i8> %in)
+ ret <32 x i8> %res
+}
+
+define <2 x i16> @pr_2xi16(<2 x i16> %in, <2 x i16> %acc) {
+; NEON-NOBF16-LABEL: pr_2xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: movi d2, #0x00ffff0000ffff
+; NEON-NOBF16-NEXT: and v0.8b, v0.8b, v2.8b
+; NEON-NOBF16-NEXT: add v0.2s, v1.2s, v0.2s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: movi d2, #0x00ffff0000ffff
+; NEON-BF16-NEXT: and v0.8b, v0.8b, v2.8b
+; NEON-BF16-NEXT: add v0.2s, v1.2s, v0.2s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: and z0.s, z0.s, #0xffff
+; SVE-NEXT: add v0.2s, v1.2s, v0.2s
+; SVE-NEXT: ret
+ %res = call <2 x i16> @llvm.vector.partial.reduce.add(<2 x i16> %acc, <2 x i16> %in)
+ ret <2 x i16> %res
+}
+
+define <4 x i16> @pr_4xi16(<4 x i16> %in, <4 x i16> %acc) {
+; NEON-NOBF16-LABEL: pr_4xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.4h, v1.4h, v0.4h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.4h, v1.4h, v0.4h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %res = call <4 x i16> @llvm.vector.partial.reduce.add(<4 x i16> %acc, <4 x i16> %in)
+ ret <4 x i16> %res
+}
+
+define <8 x i16> @pr_8xi16(<8 x i16> %in, <8 x i16> %acc) {
+; NEON-NOBF16-LABEL: pr_8xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.8h, v1.8h, v0.8h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.8h, v1.8h, v0.8h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.8h, v1.8h, v0.8h
+; SVE-NEXT: ret
+ %res = call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> %acc, <8 x i16> %in)
+ ret <8 x i16> %res
+}
+
+define <16 x i16> @pr_16xi16(<16 x i16> %in, <16 x i16> %acc) {
+; NEON-NOBF16-LABEL: pr_16xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v1.8h, v3.8h, v1.8h
+; NEON-NOBF16-NEXT: add v0.8h, v2.8h, v0.8h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_16xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v1.8h, v3.8h, v1.8h
+; NEON-BF16-NEXT: add v0.8h, v2.8h, v0.8h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_16xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: add v1.8h, v3.8h, v1.8h
+; SVE-NEXT: add v0.8h, v2.8h, v0.8h
+; SVE-NEXT: ret
+ %res = call <16 x i16> @llvm.vector.partial.reduce.add(<16 x i16> %acc, <16 x i16> %in)
+ ret <16 x i16> %res
+}
+
+define <2 x i32> @pr_2xi32(<2 x i32> %in, <2 x i32> %acc) {
+; NEON-NOBF16-LABEL: pr_2xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.2s, v1.2s, v0.2s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.2s, v1.2s, v0.2s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.2s, v1.2s, v0.2s
+; SVE-NEXT: ret
+ %res = call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> %acc, <2 x i32> %in)
+ ret <2 x i32> %res
+}
+
+define <4 x i32> @pr_4xi32(<4 x i32> %in, <4 x i32> %acc) {
+; NEON-NOBF16-LABEL: pr_4xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.4s, v1.4s, v0.4s
+; SVE-NEXT: ret
+ %res = call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> %acc, <4 x i32> %in)
+ ret <4 x i32> %res
+}
+
+define <8 x i32> @pr_8xi32(<8 x i32> %in, <8 x i32> %acc) {
+; NEON-NOBF16-LABEL: pr_8xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v1.4s, v3.4s, v1.4s
+; NEON-NOBF16-NEXT: add v0.4s, v2.4s, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v1.4s, v3.4s, v1.4s
+; NEON-BF16-NEXT: add v0.4s, v2.4s, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: add v1.4s, v3.4s, v1.4s
+; SVE-NEXT: add v0.4s, v2.4s, v0.4s
+; SVE-NEXT: ret
+ %res = call <8 x i32> @llvm.vector.partial.reduce.add(<8 x i32> %acc, <8 x i32> %in)
+ ret <8 x i32> %res
+}
+
+define <2 x i64> @pr_2xi64(<2 x i64> %in, <2 x i64> %acc) {
+; NEON-NOBF16-LABEL: pr_2xi64:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v0.2d, v1.2d, v0.2d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xi64:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v0.2d, v1.2d, v0.2d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xi64:
+; SVE: // %bb.0:
+; SVE-NEXT: add v0.2d, v1.2d, v0.2d
+; SVE-NEXT: ret
+ %res = call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> %acc, <2 x i64> %in)
+ ret <2 x i64> %res
+}
+
+define <4 x i64> @pr_4xi64(<4 x i64> %in, <4 x i64> %acc) {
+; NEON-NOBF16-LABEL: pr_4xi64:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add v1.2d, v3.2d, v1.2d
+; NEON-NOBF16-NEXT: add v0.2d, v2.2d, v0.2d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xi64:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add v1.2d, v3.2d, v1.2d
+; NEON-BF16-NEXT: add v0.2d, v2.2d, v0.2d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xi64:
+; SVE: // %bb.0:
+; SVE-NEXT: add v1.2d, v3.2d, v1.2d
+; SVE-NEXT: add v0.2d, v2.2d, v0.2d
+; SVE-NEXT: ret
+ %res = call <4 x i64> @llvm.vector.partial.reduce.add(<4 x i64> %acc, <4 x i64> %in)
+ ret <4 x i64> %res
+}
+
+define <vscale x 8 x i8> @pr_nx8xi8(<vscale x 8 x i8> %in, <vscale x 8 x i8> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: uaddwb z0.h, z1.h, z0.b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: uaddwb z0.h, z1.h, z0.b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: uaddwb z0.h, z1.h, z0.b
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x i8> @llvm.vector.partial.reduce.add(<vscale x 8 x i8> %acc, <vscale x 8 x i8> %in)
+ ret <vscale x 8 x i8> %res
+}
+
+define <vscale x 16 x i8> @pr_nx16xi8(<vscale x 16 x i8> %in, <vscale x 16 x i8> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx16xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z0.b, z1.b, z0.b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx16xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z0.b, z1.b, z0.b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx16xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: add z0.b, z1.b, z0.b
+; SVE-NEXT: ret
+ %res = call <vscale x 16 x i8> @llvm.vector.partial.reduce.add(<vscale x 16 x i8> %acc, <vscale x 16 x i8> %in)
+ ret <vscale x 16 x i8> %res
+}
+
+define <vscale x 32 x i8> @pr_nx32xi8(<vscale x 32 x i8> %in, <vscale x 32 x i8> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx32xi8:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z1.b, z3.b, z1.b
+; NEON-NOBF16-NEXT: add z0.b, z2.b, z0.b
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx32xi8:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z1.b, z3.b, z1.b
+; NEON-BF16-NEXT: add z0.b, z2.b, z0.b
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx32xi8:
+; SVE: // %bb.0:
+; SVE-NEXT: add z1.b, z3.b, z1.b
+; SVE-NEXT: add z0.b, z2.b, z0.b
+; SVE-NEXT: ret
+ %res = call <vscale x 32 x i8> @llvm.vector.partial.reduce.add(<vscale x 32 x i8> %acc, <vscale x 32 x i8> %in)
+ ret <vscale x 32 x i8> %res
+}
+
+define <vscale x 4 x i16> @pr_nx4xi16(<vscale x 4 x i16> %in, <vscale x 4 x i16> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: uaddwb z0.s, z1.s, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: uaddwb z0.s, z1.s, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: uaddwb z0.s, z1.s, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x i16> @llvm.vector.partial.reduce.add(<vscale x 4 x i16> %acc, <vscale x 4 x i16> %in)
+ ret <vscale x 4 x i16> %res
+}
+
+define <vscale x 8 x i16> @pr_nx8xi16(<vscale x 8 x i16> %in, <vscale x 8 x i16> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z0.h, z1.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z0.h, z1.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: add z0.h, z1.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 8 x i16> %in)
+ ret <vscale x 8 x i16> %res
+}
+
+define <vscale x 16 x i16> @pr_nx16xi16(<vscale x 16 x i16> %in, <vscale x 16 x i16> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx16xi16:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z1.h, z3.h, z1.h
+; NEON-NOBF16-NEXT: add z0.h, z2.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx16xi16:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z1.h, z3.h, z1.h
+; NEON-BF16-NEXT: add z0.h, z2.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx16xi16:
+; SVE: // %bb.0:
+; SVE-NEXT: add z1.h, z3.h, z1.h
+; SVE-NEXT: add z0.h, z2.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 16 x i16> @llvm.vector.partial.reduce.add(<vscale x 16 x i16> %acc, <vscale x 16 x i16> %in)
+ ret <vscale x 16 x i16> %res
+}
+
+define <vscale x 2 x i32> @pr_nx2xi32(<vscale x 2 x i32> %in, <vscale x 2 x i32> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx2xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: adr z0.d, [z1.d, z0.d, uxtw]
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx2xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: adr z0.d, [z1.d, z0.d, uxtw]
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx2xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: adr z0.d, [z1.d, z0.d, uxtw]
+; SVE-NEXT: ret
+ %res = call <vscale x 2 x i32> @llvm.vector.partial.reduce.add(<vscale x 2 x i32> %acc, <vscale x 2 x i32> %in)
+ ret <vscale x 2 x i32> %res
+}
+
+define <vscale x 4 x i32> @pr_nx4xi32(<vscale x 4 x i32> %in, <vscale x 4 x i32> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z0.s, z1.s, z0.s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z0.s, z1.s, z0.s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: add z0.s, z1.s, z0.s
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 4 x i32> %in)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 8 x i32> @pr_nx8xi32(<vscale x 8 x i32> %in, <vscale x 8 x i32> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xi32:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z1.s, z3.s, z1.s
+; NEON-NOBF16-NEXT: add z0.s, z2.s, z0.s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xi32:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z1.s, z3.s, z1.s
+; NEON-BF16-NEXT: add z0.s, z2.s, z0.s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xi32:
+; SVE: // %bb.0:
+; SVE-NEXT: add z1.s, z3.s, z1.s
+; SVE-NEXT: add z0.s, z2.s, z0.s
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x i32> @llvm.vector.partial.reduce.add(<vscale x 8 x i32> %acc, <vscale x 8 x i32> %in)
+ ret <vscale x 8 x i32> %res
+}
+
+define <vscale x 2 x i64> @pr_nx2xi64(<vscale x 2 x i64> %in, <vscale x 2 x i64> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx2xi64:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z0.d, z1.d, z0.d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx2xi64:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z0.d, z1.d, z0.d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx2xi64:
+; SVE: // %bb.0:
+; SVE-NEXT: add z0.d, z1.d, z0.d
+; SVE-NEXT: ret
+ %res = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 2 x i64> %in)
+ ret <vscale x 2 x i64> %res
+}
+
+define <vscale x 4 x i64> @pr_nx4xi64(<vscale x 4 x i64> %in, <vscale x 4 x i64> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xi64:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: add z1.d, z3.d, z1.d
+; NEON-NOBF16-NEXT: add z0.d, z2.d, z0.d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xi64:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: add z1.d, z3.d, z1.d
+; NEON-BF16-NEXT: add z0.d, z2.d, z0.d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xi64:
+; SVE: // %bb.0:
+; SVE-NEXT: add z1.d, z3.d, z1.d
+; SVE-NEXT: add z0.d, z2.d, z0.d
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x i64> @llvm.vector.partial.reduce.add(<vscale x 4 x i64> %acc, <vscale x 4 x i64> %in)
+ ret <vscale x 4 x i64> %res
+}
+
+;; FP
+define <2 x half> @pr_2xhalf(<2 x half> %in, <2 x half> %acc) {
+; NEON-NOBF16-LABEL: pr_2xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fcvtl v0.4s, v0.4h
+; NEON-NOBF16-NEXT: fcvtl v1.4s, v1.4h
+; NEON-NOBF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: fcvtn v0.4h, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fcvtl v0.4s, v0.4h
+; NEON-BF16-NEXT: fcvtl v1.4s, v1.4h
+; NEON-BF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: fcvtn v0.4h, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %res = call <2 x half> @llvm.vector.partial.reduce.fadd(<2 x half> %acc, <2 x half> %in)
+ ret <2 x half> %res
+}
+
+define <4 x half> @pr_4xhalf(<4 x half> %in, <4 x half> %acc) {
+; NEON-NOBF16-LABEL: pr_4xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fcvtl v0.4s, v0.4h
+; NEON-NOBF16-NEXT: fcvtl v1.4s, v1.4h
+; NEON-NOBF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: fcvtn v0.4h, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fcvtl v0.4s, v0.4h
+; NEON-BF16-NEXT: fcvtl v1.4s, v1.4h
+; NEON-BF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: fcvtn v0.4h, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.4h, v1.4h, v0.4h
+; SVE-NEXT: ret
+ %res = call <4 x half> @llvm.vector.partial.reduce.fadd(<4 x half> %acc, <4 x half> %in)
+ ret <4 x half> %res
+}
+
+define <8 x half> @pr_8xhalf(<8 x half> %in, <8 x half> %acc) {
+; NEON-NOBF16-LABEL: pr_8xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fcvtl v2.4s, v0.4h
+; NEON-NOBF16-NEXT: fcvtl v3.4s, v1.4h
+; NEON-NOBF16-NEXT: fcvtl2 v0.4s, v0.8h
+; NEON-NOBF16-NEXT: fcvtl2 v1.4s, v1.8h
+; NEON-NOBF16-NEXT: fadd v2.4s, v3.4s, v2.4s
+; NEON-NOBF16-NEXT: fadd v1.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: fcvtn v0.4h, v2.4s
+; NEON-NOBF16-NEXT: fcvtn2 v0.8h, v1.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fcvtl v2.4s, v0.4h
+; NEON-BF16-NEXT: fcvtl v3.4s, v1.4h
+; NEON-BF16-NEXT: fcvtl2 v0.4s, v0.8h
+; NEON-BF16-NEXT: fcvtl2 v1.4s, v1.8h
+; NEON-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
+; NEON-BF16-NEXT: fadd v1.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: fcvtn v0.4h, v2.4s
+; NEON-BF16-NEXT: fcvtn2 v0.8h, v1.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.8h, v1.8h, v0.8h
+; SVE-NEXT: ret
+ %res = call <8 x half> @llvm.vector.partial.reduce.fadd(<8 x half> %acc, <8 x half> %in)
+ ret <8 x half> %res
+}
+
+define <16 x half> @pr_16xhalf(<16 x half> %in, <16 x half> %acc) {
+; NEON-NOBF16-LABEL: pr_16xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fcvtl v4.4s, v2.4h
+; NEON-NOBF16-NEXT: fcvtl v5.4s, v0.4h
+; NEON-NOBF16-NEXT: fcvtl v6.4s, v1.4h
+; NEON-NOBF16-NEXT: fcvtl v7.4s, v3.4h
+; NEON-NOBF16-NEXT: fcvtl2 v0.4s, v0.8h
+; NEON-NOBF16-NEXT: fcvtl2 v2.4s, v2.8h
+; NEON-NOBF16-NEXT: fcvtl2 v1.4s, v1.8h
+; NEON-NOBF16-NEXT: fcvtl2 v3.4s, v3.8h
+; NEON-NOBF16-NEXT: fadd v4.4s, v4.4s, v5.4s
+; NEON-NOBF16-NEXT: fadd v5.4s, v7.4s, v6.4s
+; NEON-NOBF16-NEXT: fadd v2.4s, v2.4s, v0.4s
+; NEON-NOBF16-NEXT: fadd v3.4s, v3.4s, v1.4s
+; NEON-NOBF16-NEXT: fcvtn v0.4h, v4.4s
+; NEON-NOBF16-NEXT: fcvtn v1.4h, v5.4s
+; NEON-NOBF16-NEXT: fcvtn2 v0.8h, v2.4s
+; NEON-NOBF16-NEXT: fcvtn2 v1.8h, v3.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_16xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fcvtl v4.4s, v2.4h
+; NEON-BF16-NEXT: fcvtl v5.4s, v0.4h
+; NEON-BF16-NEXT: fcvtl v6.4s, v1.4h
+; NEON-BF16-NEXT: fcvtl v7.4s, v3.4h
+; NEON-BF16-NEXT: fcvtl2 v0.4s, v0.8h
+; NEON-BF16-NEXT: fcvtl2 v2.4s, v2.8h
+; NEON-BF16-NEXT: fcvtl2 v1.4s, v1.8h
+; NEON-BF16-NEXT: fcvtl2 v3.4s, v3.8h
+; NEON-BF16-NEXT: fadd v4.4s, v4.4s, v5.4s
+; NEON-BF16-NEXT: fadd v5.4s, v7.4s, v6.4s
+; NEON-BF16-NEXT: fadd v2.4s, v2.4s, v0.4s
+; NEON-BF16-NEXT: fadd v3.4s, v3.4s, v1.4s
+; NEON-BF16-NEXT: fcvtn v0.4h, v4.4s
+; NEON-BF16-NEXT: fcvtn v1.4h, v5.4s
+; NEON-BF16-NEXT: fcvtn2 v0.8h, v2.4s
+; NEON-BF16-NEXT: fcvtn2 v1.8h, v3.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_16xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v1.8h, v3.8h, v1.8h
+; SVE-NEXT: fadd v0.8h, v2.8h, v0.8h
+; SVE-NEXT: ret
+ %res = call <16 x half> @llvm.vector.partial.reduce.fadd(<16 x half> %acc, <16 x half> %in)
+ ret <16 x half> %res
+}
+
+define <2 x bfloat> @pr_2xbfloat(<2 x bfloat> %in, <2 x bfloat> %acc) {
+; NEON-NOBF16-LABEL: pr_2xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: shll v0.4s, v0.4h, #16
+; NEON-NOBF16-NEXT: shll v1.4s, v1.4h, #16
+; NEON-NOBF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: movi v1.4s, #1
+; NEON-NOBF16-NEXT: ushr v2.4s, v0.4s, #16
+; NEON-NOBF16-NEXT: and v1.16b, v2.16b, v1.16b
+; NEON-NOBF16-NEXT: movi v2.4s, #127, msl #8
+; NEON-NOBF16-NEXT: add v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: shll v0.4s, v0.4h, #16
+; NEON-BF16-NEXT: shll v1.4s, v1.4h, #16
+; NEON-BF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: // kill: def $d1 killed $d1 def $z1
+; SVE-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %res = call <2 x bfloat> @llvm.vector.partial.reduce.fadd(<2 x bfloat> %acc, <2 x bfloat> %in)
+ ret <2 x bfloat> %res
+}
+
+define <4 x bfloat> @pr_4xbfloat(<4 x bfloat> %in, <4 x bfloat> %acc) {
+; NEON-NOBF16-LABEL: pr_4xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: shll v0.4s, v0.4h, #16
+; NEON-NOBF16-NEXT: shll v1.4s, v1.4h, #16
+; NEON-NOBF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: movi v1.4s, #1
+; NEON-NOBF16-NEXT: ushr v2.4s, v0.4s, #16
+; NEON-NOBF16-NEXT: and v1.16b, v2.16b, v1.16b
+; NEON-NOBF16-NEXT: movi v2.4s, #127, msl #8
+; NEON-NOBF16-NEXT: add v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: addhn v0.4h, v0.4s, v2.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: shll v0.4s, v0.4h, #16
+; NEON-BF16-NEXT: shll v1.4s, v1.4h, #16
+; NEON-BF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: bfcvtn v0.4h, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h, vl4
+; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
+; SVE-NEXT: // kill: def $d1 killed $d1 def $z1
+; SVE-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
+; SVE-NEXT: ret
+ %res = call <4 x bfloat> @llvm.vector.partial.reduce.fadd(<4 x bfloat> %acc, <4 x bfloat> %in)
+ ret <4 x bfloat> %res
+}
+
+define <8 x bfloat> @pr_8xbfloat(<8 x bfloat> %in, <8 x bfloat> %acc) {
+; NEON-NOBF16-LABEL: pr_8xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: shll v3.4s, v0.4h, #16
+; NEON-NOBF16-NEXT: shll v4.4s, v1.4h, #16
+; NEON-NOBF16-NEXT: shll2 v0.4s, v0.8h, #16
+; NEON-NOBF16-NEXT: shll2 v1.4s, v1.8h, #16
+; NEON-NOBF16-NEXT: movi v2.4s, #1
+; NEON-NOBF16-NEXT: movi v5.4s, #127, msl #8
+; NEON-NOBF16-NEXT: fadd v3.4s, v4.4s, v3.4s
+; NEON-NOBF16-NEXT: fadd v1.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: ushr v0.4s, v3.4s, #16
+; NEON-NOBF16-NEXT: ushr v4.4s, v1.4s, #16
+; NEON-NOBF16-NEXT: and v0.16b, v0.16b, v2.16b
+; NEON-NOBF16-NEXT: and v2.16b, v4.16b, v2.16b
+; NEON-NOBF16-NEXT: add v0.4s, v0.4s, v3.4s
+; NEON-NOBF16-NEXT: add v1.4s, v2.4s, v1.4s
+; NEON-NOBF16-NEXT: addhn v0.4h, v0.4s, v5.4s
+; NEON-NOBF16-NEXT: addhn2 v0.8h, v1.4s, v5.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: shll v2.4s, v0.4h, #16
+; NEON-BF16-NEXT: shll v3.4s, v1.4h, #16
+; NEON-BF16-NEXT: shll2 v0.4s, v0.8h, #16
+; NEON-BF16-NEXT: shll2 v1.4s, v1.8h, #16
+; NEON-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s
+; NEON-BF16-NEXT: fadd v1.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: bfcvtn v0.4h, v2.4s
+; NEON-BF16-NEXT: bfcvtn2 v0.8h, v1.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: ret
+ %res = call <8 x bfloat> @llvm.vector.partial.reduce.fadd(<8 x bfloat> %acc, <8 x bfloat> %in)
+ ret <8 x bfloat> %res
+}
+
+define <16 x bfloat> @pr_16xbfloat(<16 x bfloat> %in, <16 x bfloat> %acc) {
+; NEON-NOBF16-LABEL: pr_16xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: shll v4.4s, v2.4h, #16
+; NEON-NOBF16-NEXT: shll v5.4s, v0.4h, #16
+; NEON-NOBF16-NEXT: shll v6.4s, v1.4h, #16
+; NEON-NOBF16-NEXT: shll v7.4s, v3.4h, #16
+; NEON-NOBF16-NEXT: shll2 v0.4s, v0.8h, #16
+; NEON-NOBF16-NEXT: shll2 v2.4s, v2.8h, #16
+; NEON-NOBF16-NEXT: shll2 v1.4s, v1.8h, #16
+; NEON-NOBF16-NEXT: shll2 v3.4s, v3.8h, #16
+; NEON-NOBF16-NEXT: movi v17.4s, #127, msl #8
+; NEON-NOBF16-NEXT: fadd v4.4s, v4.4s, v5.4s
+; NEON-NOBF16-NEXT: fadd v5.4s, v7.4s, v6.4s
+; NEON-NOBF16-NEXT: fadd v2.4s, v2.4s, v0.4s
+; NEON-NOBF16-NEXT: movi v0.4s, #1
+; NEON-NOBF16-NEXT: fadd v3.4s, v3.4s, v1.4s
+; NEON-NOBF16-NEXT: ushr v6.4s, v4.4s, #16
+; NEON-NOBF16-NEXT: ushr v1.4s, v5.4s, #16
+; NEON-NOBF16-NEXT: ushr v7.4s, v2.4s, #16
+; NEON-NOBF16-NEXT: ushr v16.4s, v3.4s, #16
+; NEON-NOBF16-NEXT: and v6.16b, v6.16b, v0.16b
+; NEON-NOBF16-NEXT: and v1.16b, v1.16b, v0.16b
+; NEON-NOBF16-NEXT: add v4.4s, v6.4s, v4.4s
+; NEON-NOBF16-NEXT: and v6.16b, v16.16b, v0.16b
+; NEON-NOBF16-NEXT: add v1.4s, v1.4s, v5.4s
+; NEON-NOBF16-NEXT: and v5.16b, v7.16b, v0.16b
+; NEON-NOBF16-NEXT: addhn v0.4h, v4.4s, v17.4s
+; NEON-NOBF16-NEXT: add v3.4s, v6.4s, v3.4s
+; NEON-NOBF16-NEXT: addhn v1.4h, v1.4s, v17.4s
+; NEON-NOBF16-NEXT: add v2.4s, v5.4s, v2.4s
+; NEON-NOBF16-NEXT: addhn2 v0.8h, v2.4s, v17.4s
+; NEON-NOBF16-NEXT: addhn2 v1.8h, v3.4s, v17.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_16xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: shll v4.4s, v2.4h, #16
+; NEON-BF16-NEXT: shll v5.4s, v0.4h, #16
+; NEON-BF16-NEXT: shll v6.4s, v1.4h, #16
+; NEON-BF16-NEXT: shll v7.4s, v3.4h, #16
+; NEON-BF16-NEXT: shll2 v0.4s, v0.8h, #16
+; NEON-BF16-NEXT: shll2 v2.4s, v2.8h, #16
+; NEON-BF16-NEXT: shll2 v1.4s, v1.8h, #16
+; NEON-BF16-NEXT: shll2 v3.4s, v3.8h, #16
+; NEON-BF16-NEXT: fadd v4.4s, v4.4s, v5.4s
+; NEON-BF16-NEXT: fadd v5.4s, v7.4s, v6.4s
+; NEON-BF16-NEXT: fadd v2.4s, v2.4s, v0.4s
+; NEON-BF16-NEXT: fadd v3.4s, v3.4s, v1.4s
+; NEON-BF16-NEXT: bfcvtn v0.4h, v4.4s
+; NEON-BF16-NEXT: bfcvtn v1.4h, v5.4s
+; NEON-BF16-NEXT: bfcvtn2 v0.8h, v2.4s
+; NEON-BF16-NEXT: bfcvtn2 v1.8h, v3.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_16xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.h, vl8
+; SVE-NEXT: // kill: def $q1 killed $q1 def $z1
+; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
+; SVE-NEXT: // kill: def $q3 killed $q3 def $z3
+; SVE-NEXT: // kill: def $q2 killed $q2 def $z2
+; SVE-NEXT: bfadd z0.h, p0/m, z0.h, z2.h
+; SVE-NEXT: bfadd z1.h, p0/m, z1.h, z3.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE-NEXT: ret
+ %res = call <16 x bfloat> @llvm.vector.partial.reduce.fadd(<16 x bfloat> %acc, <16 x bfloat> %in)
+ ret <16 x bfloat> %res
+}
+
+define <2 x float> @pr_2xfloat(<2 x float> %in, <2 x float> %acc) {
+; NEON-NOBF16-LABEL: pr_2xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd v0.2s, v1.2s, v0.2s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd v0.2s, v1.2s, v0.2s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.2s, v1.2s, v0.2s
+; SVE-NEXT: ret
+ %res = call <2 x float> @llvm.vector.partial.reduce.fadd(<2 x float> %acc, <2 x float> %in)
+ ret <2 x float> %res
+}
+
+define <4 x float> @pr_4xfloat(<4 x float> %in, <4 x float> %acc) {
+; NEON-NOBF16-LABEL: pr_4xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd v0.4s, v1.4s, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.4s, v1.4s, v0.4s
+; SVE-NEXT: ret
+ %res = call <4 x float> @llvm.vector.partial.reduce.fadd(<4 x float> %acc, <4 x float> %in)
+ ret <4 x float> %res
+}
+
+define <8 x float> @pr_8xfloat(<8 x float> %in, <8 x float> %acc) {
+; NEON-NOBF16-LABEL: pr_8xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd v1.4s, v3.4s, v1.4s
+; NEON-NOBF16-NEXT: fadd v0.4s, v2.4s, v0.4s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_8xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd v1.4s, v3.4s, v1.4s
+; NEON-BF16-NEXT: fadd v0.4s, v2.4s, v0.4s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_8xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v1.4s, v3.4s, v1.4s
+; SVE-NEXT: fadd v0.4s, v2.4s, v0.4s
+; SVE-NEXT: ret
+ %res = call <8 x float> @llvm.vector.partial.reduce.fadd(<8 x float> %acc, <8 x float> %in)
+ ret <8 x float> %res
+}
+
+define <2 x double> @pr_2xdouble(<2 x double> %in, <2 x double> %acc) {
+; NEON-NOBF16-LABEL: pr_2xdouble:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd v0.2d, v1.2d, v0.2d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_2xdouble:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd v0.2d, v1.2d, v0.2d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_2xdouble:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v0.2d, v1.2d, v0.2d
+; SVE-NEXT: ret
+ %res = call <2 x double> @llvm.vector.partial.reduce.fadd(<2 x double> %acc, <2 x double> %in)
+ ret <2 x double> %res
+}
+
+define <4 x double> @pr_4xdouble(<4 x double> %in, <4 x double> %acc) {
+; NEON-NOBF16-LABEL: pr_4xdouble:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd v1.2d, v3.2d, v1.2d
+; NEON-NOBF16-NEXT: fadd v0.2d, v2.2d, v0.2d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_4xdouble:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd v1.2d, v3.2d, v1.2d
+; NEON-BF16-NEXT: fadd v0.2d, v2.2d, v0.2d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_4xdouble:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd v1.2d, v3.2d, v1.2d
+; SVE-NEXT: fadd v0.2d, v2.2d, v0.2d
+; SVE-NEXT: ret
+ %res = call <4 x double> @llvm.vector.partial.reduce.fadd(<4 x double> %acc, <4 x double> %in)
+ ret <4 x double> %res
+}
+
+define <vscale x 4 x half> @pr_nx4xhalf(<vscale x 4 x half> %in, <vscale x 4 x half> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: ptrue p0.s
+; NEON-NOBF16-NEXT: fadd z0.h, p0/m, z0.h, z1.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: ptrue p0.s
+; NEON-BF16-NEXT: fadd z0.h, p0/m, z0.h, z1.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: fadd z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x half> @llvm.vector.partial.reduce.fadd(<vscale x 4 x half> %acc, <vscale x 4 x half> %in)
+ ret <vscale x 4 x half> %res
+}
+
+define <vscale x 8 x half> @pr_nx8xhalf(<vscale x 8 x half> %in, <vscale x 8 x half> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z0.h, z1.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z0.h, z1.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z0.h, z1.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x half> @llvm.vector.partial.reduce.fadd(<vscale x 8 x half> %acc, <vscale x 8 x half> %in)
+ ret <vscale x 8 x half> %res
+}
+
+define <vscale x 16 x half> @pr_nx16xhalf(<vscale x 16 x half> %in, <vscale x 16 x half> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx16xhalf:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z1.h, z3.h, z1.h
+; NEON-NOBF16-NEXT: fadd z0.h, z2.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx16xhalf:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z1.h, z3.h, z1.h
+; NEON-BF16-NEXT: fadd z0.h, z2.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx16xhalf:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z1.h, z3.h, z1.h
+; SVE-NEXT: fadd z0.h, z2.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 16 x half> @llvm.vector.partial.reduce.fadd(<vscale x 16 x half> %acc, <vscale x 16 x half> %in)
+ ret <vscale x 16 x half> %res
+}
+
+define <vscale x 4 x bfloat> @pr_nx4xbfloat(<vscale x 4 x bfloat> %in, <vscale x 4 x bfloat> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: ptrue p0.s
+; NEON-NOBF16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: ptrue p0.s
+; NEON-BF16-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: bfadd z0.h, p0/m, z0.h, z1.h
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 4 x bfloat> %acc, <vscale x 4 x bfloat> %in)
+ ret <vscale x 4 x bfloat> %res
+}
+
+define <vscale x 8 x bfloat> @pr_nx8xbfloat(<vscale x 8 x bfloat> %in, <vscale x 8 x bfloat> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: bfadd z0.h, z1.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: bfadd z0.h, z1.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: bfadd z0.h, z1.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %in)
+ ret <vscale x 8 x bfloat> %res
+}
+
+define <vscale x 16 x bfloat> @pr_nx16xbfloat(<vscale x 16 x bfloat> %in, <vscale x 16 x bfloat> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx16xbfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: bfadd z1.h, z3.h, z1.h
+; NEON-NOBF16-NEXT: bfadd z0.h, z2.h, z0.h
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx16xbfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: bfadd z1.h, z3.h, z1.h
+; NEON-BF16-NEXT: bfadd z0.h, z2.h, z0.h
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx16xbfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: bfadd z1.h, z3.h, z1.h
+; SVE-NEXT: bfadd z0.h, z2.h, z0.h
+; SVE-NEXT: ret
+ %res = call <vscale x 16 x bfloat> @llvm.vector.partial.reduce.fadd(<vscale x 16 x bfloat> %acc, <vscale x 16 x bfloat> %in)
+ ret <vscale x 16 x bfloat> %res
+}
+
+define <vscale x 2 x float> @pr_nx2xfloat(<vscale x 2 x float> %in, <vscale x 2 x float> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx2xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: ptrue p0.d
+; NEON-NOBF16-NEXT: fadd z0.s, p0/m, z0.s, z1.s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx2xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: ptrue p0.d
+; NEON-BF16-NEXT: fadd z0.s, p0/m, z0.s, z1.s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx2xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: fadd z0.s, p0/m, z0.s, z1.s
+; SVE-NEXT: ret
+ %res = call <vscale x 2 x float> @llvm.vector.partial.reduce.fadd(<vscale x 2 x float> %acc, <vscale x 2 x float> %in)
+ ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x float> @pr_nx4xfloat(<vscale x 4 x float> %in, <vscale x 4 x float> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z0.s, z1.s, z0.s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z0.s, z1.s, z0.s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z0.s, z1.s, z0.s
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x float> @llvm.vector.partial.reduce.fadd(<vscale x 4 x float> %acc, <vscale x 4 x float> %in)
+ ret <vscale x 4 x float> %res
+}
+
+define <vscale x 8 x float> @pr_nx8xfloat(<vscale x 8 x float> %in, <vscale x 8 x float> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx8xfloat:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z1.s, z3.s, z1.s
+; NEON-NOBF16-NEXT: fadd z0.s, z2.s, z0.s
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx8xfloat:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z1.s, z3.s, z1.s
+; NEON-BF16-NEXT: fadd z0.s, z2.s, z0.s
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx8xfloat:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z1.s, z3.s, z1.s
+; SVE-NEXT: fadd z0.s, z2.s, z0.s
+; SVE-NEXT: ret
+ %res = call <vscale x 8 x float> @llvm.vector.partial.reduce.fadd(<vscale x 8 x float> %acc, <vscale x 8 x float> %in)
+ ret <vscale x 8 x float> %res
+}
+
+define <vscale x 2 x double> @pr_nx2xdouble(<vscale x 2 x double> %in, <vscale x 2 x double> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx2xdouble:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z0.d, z1.d, z0.d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx2xdouble:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z0.d, z1.d, z0.d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx2xdouble:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z0.d, z1.d, z0.d
+; SVE-NEXT: ret
+ %res = call <vscale x 2 x double> @llvm.vector.partial.reduce.fadd(<vscale x 2 x double> %acc, <vscale x 2 x double> %in)
+ ret <vscale x 2 x double> %res
+}
+
+define <vscale x 4 x double> @pr_nx4xdouble(<vscale x 4 x double> %in, <vscale x 4 x double> %acc) #0 {
+; NEON-NOBF16-LABEL: pr_nx4xdouble:
+; NEON-NOBF16: // %bb.0:
+; NEON-NOBF16-NEXT: fadd z1.d, z3.d, z1.d
+; NEON-NOBF16-NEXT: fadd z0.d, z2.d, z0.d
+; NEON-NOBF16-NEXT: ret
+;
+; NEON-BF16-LABEL: pr_nx4xdouble:
+; NEON-BF16: // %bb.0:
+; NEON-BF16-NEXT: fadd z1.d, z3.d, z1.d
+; NEON-BF16-NEXT: fadd z0.d, z2.d, z0.d
+; NEON-BF16-NEXT: ret
+;
+; SVE-LABEL: pr_nx4xdouble:
+; SVE: // %bb.0:
+; SVE-NEXT: fadd z1.d, z3.d, z1.d
+; SVE-NEXT: fadd z0.d, z2.d, z0.d
+; SVE-NEXT: ret
+ %res = call <vscale x 4 x double> @llvm.vector.partial.reduce.fadd(<vscale x 4 x double> %acc, <vscale x 4 x double> %in)
+ ret <vscale x 4 x double> %res
+}
+
+attributes #0 = { nounwind "target-features"="+sve2,+bf16,+sve-b16b16" vscale_range(1,16) }
More information about the llvm-commits
mailing list