[llvm] [AArch64] Lower and price partial reductions without dotprod (PR #214636)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 21:43:48 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/214636
>From 942d36da5f57614aa7e9dc9b486bb2c1ee88c62a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 03:36:35 +0000
Subject: [PATCH 1/4] [AArch64] Add tests for wider partial reductions
---
.../neon-partial-reduce-dot-product.ll | 207 ++++++++++++++++++
1 file changed, 207 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 5213e11205e3c..c728c4891f221 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1799,3 +1799,210 @@ define <2 x i64> @partial_reduce_sext_cmp_i32tov2i64(<2 x i64> %acc, <4 x i32> %
ret <2 x i64> %partial.reduce
}
+define <2 x i64> @partial_reduce_sext_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %in) {
+; CHECK-NODOT-LABEL: partial_reduce_sext_v16i8_v2i64:
+; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0
+; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-NODOT-NEXT: sshll v3.4s, v2.4h, #0
+; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-NODOT-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v2.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v1.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: ret
+;
+; CHECK-DOT-LABEL: partial_reduce_sext_v16i8_v2i64:
+; CHECK-DOT: // %bb.0:
+; CHECK-DOT-NEXT: movi v2.16b, #1
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-NEXT: sadalp v0.2d, v3.4s
+; CHECK-DOT-NEXT: ret
+;
+; CHECK-DOT-I8MM-LABEL: partial_reduce_sext_v16i8_v2i64:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v2.16b, #1
+; CHECK-DOT-I8MM-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: sdot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v3.4s
+; CHECK-DOT-I8MM-NEXT: ret
+ %ext = sext <16 x i8> %in to <16 x i64>
+ %r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %ext)
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @partial_reduce_umull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-NODOT-LABEL: partial_reduce_umull_v16i8_v2i64:
+; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: umull v3.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: umull2 v1.8h, v1.16b, v2.16b
+; CHECK-NODOT-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NODOT-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NODOT-NEXT: ushll v2.4s, v1.4h, #0
+; CHECK-NODOT-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
+; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s
+; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v2.2s
+; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v2.4s
+; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v1.2s
+; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: ret
+;
+; CHECK-DOT-LABEL: partial_reduce_umull_v16i8_v2i64:
+; CHECK-DOT: // %bb.0:
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-NEXT: uadalp v0.2d, v3.4s
+; CHECK-DOT-NEXT: ret
+;
+; CHECK-DOT-I8MM-LABEL: partial_reduce_umull_v16i8_v2i64:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: udot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: uadalp v0.2d, v3.4s
+; CHECK-DOT-I8MM-NEXT: ret
+ %ea = zext <16 x i8> %a to <16 x i64>
+ %eb = zext <16 x i8> %b to <16 x i64>
+ %m = mul <16 x i64> %ea, %eb
+ %r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %m)
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @partial_reduce_smull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
+; CHECK-NODOT-LABEL: partial_reduce_smull_v16i8_v2i64:
+; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: smull v3.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: smull2 v1.8h, v1.16b, v2.16b
+; CHECK-NODOT-NEXT: sshll v4.4s, v3.4h, #0
+; CHECK-NODOT-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NODOT-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-NODOT-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v2.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
+; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v1.2s
+; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: ret
+;
+; CHECK-DOT-LABEL: partial_reduce_smull_v16i8_v2i64:
+; CHECK-DOT: // %bb.0:
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-NEXT: sadalp v0.2d, v3.4s
+; CHECK-DOT-NEXT: ret
+;
+; CHECK-DOT-I8MM-LABEL: partial_reduce_smull_v16i8_v2i64:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: sdot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v3.4s
+; CHECK-DOT-I8MM-NEXT: ret
+ %ea = sext <16 x i8> %a to <16 x i64>
+ %eb = sext <16 x i8> %b to <16 x i64>
+ %m = mul <16 x i64> %ea, %eb
+ %r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %m)
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @partial_reduce_sext_v8i16_v2i64(<2 x i64> %acc, <8 x i16> %in) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_v8i16_v2i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: sshll v2.4s, v1.4h, #0
+; CHECK-COMMON-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-COMMON-NEXT: saddw v0.2d, v0.2d, v2.2s
+; CHECK-COMMON-NEXT: saddw2 v0.2d, v0.2d, v2.4s
+; CHECK-COMMON-NEXT: saddw v0.2d, v0.2d, v1.2s
+; CHECK-COMMON-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-COMMON-NEXT: ret
+ %ext = sext <8 x i16> %in to <8 x i64>
+ %r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %ext)
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @partial_reduce_smull_v8i16_v2i64(<2 x i64> %acc, <8 x i16> %a, <8 x i16> %b) {
+; CHECK-COMMON-LABEL: partial_reduce_smull_v8i16_v2i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: smull v3.4s, v1.4h, v2.4h
+; CHECK-COMMON-NEXT: smull2 v1.4s, v1.8h, v2.8h
+; CHECK-COMMON-NEXT: sadalp v0.2d, v3.4s
+; CHECK-COMMON-NEXT: sadalp v0.2d, v1.4s
+; CHECK-COMMON-NEXT: ret
+ %ea = sext <8 x i16> %a to <8 x i64>
+ %eb = sext <8 x i16> %b to <8 x i64>
+ %m = mul <8 x i64> %ea, %eb
+ %r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %m)
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @udot_in_loop_8to64(ptr %p1, ptr %p2){
+; CHECK-COMMON-LABEL: udot_in_loop_8to64:
+; CHECK-COMMON: // %bb.0: // %entry
+; CHECK-COMMON-NEXT: adrp x8, .LCPI54_0
+; CHECK-COMMON-NEXT: movi v1.2d, #0000000000000000
+; CHECK-COMMON-NEXT: adrp x9, .LCPI54_2
+; CHECK-COMMON-NEXT: ldr q2, [x8, :lo12:.LCPI54_0]
+; CHECK-COMMON-NEXT: adrp x8, .LCPI54_1
+; CHECK-COMMON-NEXT: adrp x10, .LCPI54_3
+; CHECK-COMMON-NEXT: ldr q3, [x8, :lo12:.LCPI54_1]
+; CHECK-COMMON-NEXT: ldr q4, [x9, :lo12:.LCPI54_2]
+; CHECK-COMMON-NEXT: ldr q5, [x10, :lo12:.LCPI54_3]
+; CHECK-COMMON-NEXT: mov x8, xzr
+; CHECK-COMMON-NEXT: .LBB54_1: // %vector.body
+; CHECK-COMMON-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-COMMON-NEXT: ldr q6, [x0, x8]
+; CHECK-COMMON-NEXT: ldr q7, [x1, x8]
+; CHECK-COMMON-NEXT: mov v0.16b, v1.16b
+; CHECK-COMMON-NEXT: add x8, x8, #16
+; CHECK-COMMON-NEXT: tbl v16.16b, { v6.16b }, v2.16b
+; CHECK-COMMON-NEXT: tbl v17.16b, { v7.16b }, v2.16b
+; CHECK-COMMON-NEXT: tbl v18.16b, { v6.16b }, v3.16b
+; CHECK-COMMON-NEXT: tbl v19.16b, { v7.16b }, v3.16b
+; CHECK-COMMON-NEXT: cmp x8, #16
+; CHECK-COMMON-NEXT: umlal v1.2d, v16.2s, v17.2s
+; CHECK-COMMON-NEXT: umlal2 v1.2d, v16.4s, v17.4s
+; CHECK-COMMON-NEXT: tbl v16.16b, { v6.16b }, v4.16b
+; CHECK-COMMON-NEXT: tbl v17.16b, { v7.16b }, v4.16b
+; CHECK-COMMON-NEXT: tbl v6.16b, { v6.16b }, v5.16b
+; CHECK-COMMON-NEXT: tbl v7.16b, { v7.16b }, v5.16b
+; CHECK-COMMON-NEXT: umlal v1.2d, v18.2s, v19.2s
+; CHECK-COMMON-NEXT: umlal2 v1.2d, v18.4s, v19.4s
+; CHECK-COMMON-NEXT: umlal v1.2d, v16.2s, v17.2s
+; CHECK-COMMON-NEXT: umlal2 v1.2d, v16.4s, v17.4s
+; CHECK-COMMON-NEXT: umlal v1.2d, v6.2s, v7.2s
+; CHECK-COMMON-NEXT: umlal2 v1.2d, v6.4s, v7.4s
+; CHECK-COMMON-NEXT: b.ne .LBB54_1
+; CHECK-COMMON-NEXT: // %bb.2: // %end
+; CHECK-COMMON-NEXT: ret
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %acc = phi <2 x i64> [ zeroinitializer, %entry ], [ %partial.reduce, %vector.body ]
+ %gep1 = getelementptr i8, ptr %p1, i64 %index
+ %load1 = load <16 x i8>, ptr %gep1, align 16
+ %load1.wide = zext <16 x i8> %load1 to <16 x i64>
+ %gep2 = getelementptr i8, ptr %p2, i64 %index
+ %load2 = load <16 x i8>, ptr %gep2, align 16
+ %load2.wide = zext <16 x i8> %load2 to <16 x i64>
+ %mul = mul nuw nsw <16 x i64> %load1.wide, %load2.wide
+ %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v16i64(<2 x i64> %acc, <16 x i64> %mul)
+ %index.next = add nuw i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 16
+ br i1 %cmp, label %end, label %vector.body
+
+end:
+ ret <2 x i64> %acc
+}
>From 54fbd82de4613f08e4fdb0ce428091185a6f208c Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 03:59:11 +0000
Subject: [PATCH 2/4] [AArch64] Fold wider partial reductions with a ladder of
[SU]ADDLP
---
.../Target/AArch64/AArch64ISelLowering.cpp | 53 +++-
.../neon-partial-reduce-dot-product.ll | 298 +++++-------------
.../sve-fixed-length-partial-reduce.ll | 82 ++---
3 files changed, 162 insertions(+), 271 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e9fdaaab967af..68bd6cb1a78de 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1548,6 +1548,15 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MLAOps, MVT::v8i16, MVT::v16i8, Custom);
setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v8i16, Custom);
setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v4i32, Custom);
+
+ // Wider folds are a ladder of the two-way step above. +dotprod has no
+ // i16 form so that shape registers unconditionally.
+ setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v8i16, Custom);
+ if (!Subtarget->hasDotProd()) {
+ setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v16i8, Custom);
+ setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v16i8, Custom);
+ setPartialReduceMLAAction(MLAOps, MVT::v2i32, MVT::v16i8, Custom);
+ }
}
if (Subtarget->hasDotProd()) {
@@ -34866,12 +34875,13 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
return Scatter;
}
-/// Lower a PARTIAL_REDUCE_MLA node. Three cases are handled:
+/// Lower a PARTIAL_REDUCE_MLA node. The following cases are handled:
/// 1. (v2i32, v16i8): widen Acc to v4i32 and fold the high half with ADDP.
/// 2. (nx)v2i64/(nx)v16i8: accumulate in two steps via (nx)v4i32, using
/// (U|S)ADALP when available, otherwise add(add(Acc, ext(lo), ext(hi))).
/// 3. SUMLA on (v4i32, v16i8) or (v2i32, v8i8) without +i8mm: rewrite as two
/// UDOTs using the bias-128 identity sext(s) = zext(s ^ 128) - 128.
+/// 4. Wider fixed-length folds do one [SU]ADDLP rung and re-enter.
SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
@@ -34930,6 +34940,47 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
DAG.getNode(Opc, DL, ResultVT, Wide));
}
+ // Wider fixed-length integer add reductions fold with a ladder of [SU]ADDLP,
+ // one rung at a time, ending at the two-way reduction above.
+ bool IsAddMLA = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA ||
+ Op.getOpcode() == ISD::PARTIAL_REDUCE_SMLA;
+ bool NeonOwnsVT = Subtarget->isNeonAvailable() &&
+ !Subtarget->isSVEorStreamingSVEAvailable() &&
+ (OpVT.is64BitVector() || OpVT.is128BitVector()) &&
+ ResultVT.getSizeInBits() == OpVT.getSizeInBits();
+ unsigned LaneRatio =
+ ResultVT.getScalarSizeInBits() / OpVT.getScalarSizeInBits();
+
+ if (IsAddMLA && NeonOwnsVT && LaneRatio >= 4) {
+ bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
+ unsigned AddlpOpc = IsUnsigned ? AArch64ISD::UADDLP : AArch64ISD::SADDLP;
+ unsigned MullOpc = IsUnsigned ? AArch64ISD::UMULL : AArch64ISD::SMULL;
+ EVT NextVT = EVT::getVectorVT(
+ *DAG.getContext(),
+ EVT::getIntegerVT(*DAG.getContext(), OpVT.getScalarSizeInBits() * 2),
+ OpVT.getVectorNumElements() / 2);
+
+ // A pure reduction peels one rung with [SU]ADDLP and re-enters.
+ if (isOneVector(RHS))
+ return DAG.getNode(Op.getOpcode(), DL, ResultVT, Acc,
+ DAG.getNode(AddlpOpc, DL, NextVT, LHS),
+ DAG.getConstant(1, DL, NextVT));
+
+ // [us]mull widens the products by one rung and the rest of the ladder
+ // re-enters as a plain sum of them.
+ EVT HalfVT = OpVT.getHalfNumVectorElementsVT(*DAG.getContext());
+ unsigned Half = OpVT.getVectorNumElements() / 2;
+ SDValue Lo = DAG.getNode(MullOpc, DL, NextVT,
+ DAG.getExtractSubvector(DL, HalfVT, LHS, 0),
+ DAG.getExtractSubvector(DL, HalfVT, RHS, 0));
+ SDValue Hi = DAG.getNode(MullOpc, DL, NextVT,
+ DAG.getExtractSubvector(DL, HalfVT, LHS, Half),
+ DAG.getExtractSubvector(DL, HalfVT, RHS, Half));
+ SDValue One = DAG.getConstant(1, DL, NextVT);
+ SDValue Sum = DAG.getNode(Op.getOpcode(), DL, ResultVT, Acc, Lo, One);
+ return DAG.getNode(Op.getOpcode(), DL, ResultVT, Sum, Hi, One);
+ }
+
// Lower PARTIAL_REDUCE_SUMLA on targets without +i8mm using udot via
// sum(sext(LHS) * zext(RHS)) =
// sum(zext(LHS ^ 128) * zext(RHS)) - sum(128 * zext(RHS))
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index c728c4891f221..2d09baf17faf3 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -577,20 +577,12 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: udot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
-; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b
+; CHECK-NODOT-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NODOT-NEXT: umull v5.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b
-; CHECK-NODOT-NEXT: ushll v3.4s, v4.4h, #0
-; CHECK-NODOT-NEXT: ushll v5.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v5.2s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v5.4s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v2.2s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NODOT-NEXT: uadalp v4.4s, v5.8h
+; CHECK-NODOT-NEXT: uadalp v4.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_8to64:
@@ -618,20 +610,12 @@ entry:
define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NODOT-LABEL: sdot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
-; CHECK-NODOT-NEXT: smull v4.8h, v2.8b, v3.8b
+; CHECK-NODOT-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NODOT-NEXT: smull v5.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: smull2 v2.8h, v2.16b, v3.16b
-; CHECK-NODOT-NEXT: sshll v3.4s, v4.4h, #0
-; CHECK-NODOT-NEXT: sshll v5.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v4.4s, v4.8h, #0
-; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v5.2s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v5.4s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v2.2s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NODOT-NEXT: sadalp v4.4s, v5.8h
+; CHECK-NODOT-NEXT: sadalp v4.4s, v2.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: sdot_8to64:
@@ -763,12 +747,8 @@ entry:
define <4 x i32> @udot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: udot_no_bin_op:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: ushll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: ushll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v2.4h
-; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v2.8h
-; CHECK-NODOT-NEXT: uaddw v0.4s, v0.4s, v1.4h
-; CHECK-NODOT-NEXT: uaddw2 v0.4s, v0.4s, v1.8h
+; CHECK-NODOT-NEXT: uaddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: uadalp v0.4s, v1.8h
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_no_bin_op:
@@ -794,16 +774,12 @@ define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
; CHECK-NODOT-NEXT: mov x8, xzr
; CHECK-NODOT-NEXT: .LBB17_1: // %vector.body
; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NODOT-NEXT: ldr q2, [x0, x8]
-; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT: ldr q0, [x0, x8]
; CHECK-NODOT-NEXT: add x8, x8, #16
; CHECK-NODOT-NEXT: cmp x8, #16
-; CHECK-NODOT-NEXT: ushll v3.8h, v2.8b, #0
-; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v3.4h
-; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v3.8h
-; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v2.4h
-; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v2.8h
+; CHECK-NODOT-NEXT: uaddlp v2.8h, v0.16b
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT: uadalp v1.4s, v2.8h
; CHECK-NODOT-NEXT: b.ne .LBB17_1
; CHECK-NODOT-NEXT: // %bb.2: // %end
; CHECK-NODOT-NEXT: ret
@@ -860,12 +836,8 @@ end:
define <4 x i32> @sdot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: sdot_no_bin_op:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v2.4h
-; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h
-; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v1.8h
+; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: sadalp v0.4s, v1.8h
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: sdot_no_bin_op:
@@ -937,20 +909,10 @@ define <2 x i32> @sdot_no_bin_op_narrow(<2 x i32> %acc, <8 x i8> %a){
define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: udot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: ushll v3.8h, v2.8b, #0
-; CHECK-NODOT-NEXT: ushll2 v2.8h, v2.16b, #0
-; CHECK-NODOT-NEXT: ushll v4.4s, v3.4h, #0
-; CHECK-NODOT-NEXT: ushll v5.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v5.2s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v5.4s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
-; CHECK-NODOT-NEXT: uaddw v1.2d, v1.2d, v2.2s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NODOT-NEXT: uaddlp v2.8h, v2.16b
+; CHECK-NODOT-NEXT: uadalp v3.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_no_bin_op_8to64:
@@ -976,20 +938,10 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: sdot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: sshll v3.8h, v2.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v2.8h, v2.16b, #0
-; CHECK-NODOT-NEXT: sshll v4.4s, v3.4h, #0
-; CHECK-NODOT-NEXT: sshll v5.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v5.2s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v5.4s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
-; CHECK-NODOT-NEXT: saddw v1.2d, v1.2d, v2.2s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NODOT-NEXT: saddlp v2.8h, v2.16b
+; CHECK-NODOT-NEXT: sadalp v3.4s, v2.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: sdot_no_bin_op_8to64:
@@ -1235,12 +1187,8 @@ end:
define <2 x i64> @udot_16to64(<2 x i64> %acc, <8 x i16> %input){
; CHECK-COMMON-LABEL: udot_16to64:
; CHECK-COMMON: // %bb.0: // %entry
-; CHECK-COMMON-NEXT: ushll v2.4s, v1.4h, #0
-; CHECK-COMMON-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-COMMON-NEXT: uaddw v0.2d, v0.2d, v2.2s
-; CHECK-COMMON-NEXT: uaddw2 v0.2d, v0.2d, v2.4s
-; CHECK-COMMON-NEXT: uaddw v0.2d, v0.2d, v1.2s
-; CHECK-COMMON-NEXT: uaddw2 v0.2d, v0.2d, v1.4s
+; CHECK-COMMON-NEXT: uaddlp v1.4s, v1.8h
+; CHECK-COMMON-NEXT: uadalp v0.2d, v1.4s
; CHECK-COMMON-NEXT: ret
entry:
%input.wide = zext <8 x i16> %input to <8 x i64>
@@ -1251,16 +1199,13 @@ entry:
define <4 x i32> @partial_reduce_shl_sext_const_rhs6(<16 x i8> %l, <4 x i32> %part) {
; CHECK-NODOT-LABEL: partial_reduce_shl_sext_const_rhs6:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: sshll v2.8h, v0.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v0.8h, v0.16b, #0
-; CHECK-NODOT-NEXT: sshll v3.4s, v0.4h, #6
-; CHECK-NODOT-NEXT: sshll2 v4.4s, v2.8h, #6
-; CHECK-NODOT-NEXT: sshll v2.4s, v2.4h, #6
-; CHECK-NODOT-NEXT: sshll2 v0.4s, v0.8h, #6
-; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-NODOT-NEXT: add v2.4s, v4.4s, v3.4s
-; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-NODOT-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NODOT-NEXT: movi v2.8b, #64
+; CHECK-NODOT-NEXT: movi v3.16b, #64
+; CHECK-NODOT-NEXT: smull v2.8h, v0.8b, v2.8b
+; CHECK-NODOT-NEXT: smull2 v0.8h, v0.16b, v3.16b
+; CHECK-NODOT-NEXT: sadalp v1.4s, v2.8h
+; CHECK-NODOT-NEXT: sadalp v1.4s, v0.8h
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_shl_sext_const_rhs6:
@@ -1359,16 +1304,13 @@ define <4 x i32> @partial_reduce_shl_sext_non_const_rhs(<16 x i8> %l, <4 x i32>
define <4 x i32> @partial_reduce_shl_zext_const_rhs6(<16 x i8> %l, <4 x i32> %part) {
; CHECK-NODOT-LABEL: partial_reduce_shl_zext_const_rhs6:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: ushll v2.8h, v0.8b, #0
-; CHECK-NODOT-NEXT: ushll2 v0.8h, v0.16b, #0
-; CHECK-NODOT-NEXT: ushll v3.4s, v0.4h, #6
-; CHECK-NODOT-NEXT: ushll2 v4.4s, v2.8h, #6
-; CHECK-NODOT-NEXT: ushll v2.4s, v2.4h, #6
-; CHECK-NODOT-NEXT: ushll2 v0.4s, v0.8h, #6
-; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-NODOT-NEXT: add v2.4s, v4.4s, v3.4s
-; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
-; CHECK-NODOT-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NODOT-NEXT: movi v2.8b, #64
+; CHECK-NODOT-NEXT: movi v3.16b, #64
+; CHECK-NODOT-NEXT: umull v2.8h, v0.8b, v2.8b
+; CHECK-NODOT-NEXT: umull2 v0.8h, v0.16b, v3.16b
+; CHECK-NODOT-NEXT: uadalp v1.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v1.4s, v0.8h
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_shl_zext_const_rhs6:
@@ -1447,14 +1389,11 @@ define <4 x i32> @partial_reduce_shl_zext_non_const_rhs(<16 x i8> %l, <4 x i32>
define <2 x i32> @udot_v16i8tov2i32(<2 x i32> %acc, <16 x i8> %input) {
; CHECK-NODOT-LABEL: udot_v16i8tov2i32:
; CHECK-NODOT: // %bb.0: // %entry
-; CHECK-NODOT-NEXT: ushll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: ushll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: uadalp v0.2s, v2.4h
-; CHECK-NODOT-NEXT: mov d2, v2.d[1]
-; CHECK-NODOT-NEXT: uadalp v0.2s, v2.4h
-; CHECK-NODOT-NEXT: uadalp v0.2s, v1.4h
-; CHECK-NODOT-NEXT: mov d1, v1.d[1]
-; CHECK-NODOT-NEXT: uadalp v0.2s, v1.4h
+; CHECK-NODOT-NEXT: fmov d0, d0
+; CHECK-NODOT-NEXT: uaddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: uadalp v0.4s, v1.8h
+; CHECK-NODOT-NEXT: addp v0.4s, v0.4s, v0.4s
+; CHECK-NODOT-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_v16i8tov2i32:
@@ -1592,22 +1531,11 @@ entry:
define <4 x i32> @partial_reduce_zext_cmp_i8tov4i32(<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_zext_cmp_i8tov4i32:
; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: movi v3.16b, #1
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: movi v3.4s, #1
-; CHECK-NODOT-NEXT: ushll2 v2.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: ushll v4.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v5.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: and v4.16b, v4.16b, v3.16b
-; CHECK-NODOT-NEXT: and v5.16b, v5.16b, v3.16b
; CHECK-NODOT-NEXT: and v1.16b, v1.16b, v3.16b
-; CHECK-NODOT-NEXT: and v2.16b, v2.16b, v3.16b
-; CHECK-NODOT-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NODOT-NEXT: add v1.4s, v5.4s, v4.4s
-; CHECK-NODOT-NEXT: add v0.4s, v0.4s, v1.4s
-; CHECK-NODOT-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NODOT-NEXT: uaddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: uadalp v0.4s, v1.8h
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_zext_cmp_i8tov4i32:
@@ -1635,12 +1563,8 @@ define <4 x i32> @partial_reduce_sext_cmp_i8tov4i32(<4 x i32> %acc, <16 x i8> %a
; CHECK-NODOT-LABEL: partial_reduce_sext_cmp_i8tov4i32:
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v2.4h
-; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v2.8h
-; CHECK-NODOT-NEXT: saddw v0.4s, v0.4s, v1.4h
-; CHECK-NODOT-NEXT: saddw2 v0.4s, v0.4s, v1.8h
+; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: sadalp v0.4s, v1.8h
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_sext_cmp_i8tov4i32:
@@ -1665,39 +1589,13 @@ define <4 x i32> @partial_reduce_sext_cmp_i8tov4i32(<4 x i32> %acc, <16 x i8> %a
define <2 x i64> @partial_reduce_zext_cmp_i8tov2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_zext_cmp_i8tov2i64:
; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: movi v3.16b, #1
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: mov w8, #1 // =0x1
-; CHECK-NODOT-NEXT: dup v5.2d, x8
-; CHECK-NODOT-NEXT: ushll2 v2.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: ushll v3.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v4.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: ushll v6.2d, v3.2s, #0
-; CHECK-NODOT-NEXT: ushll2 v7.2d, v4.4s, #0
-; CHECK-NODOT-NEXT: ushll v4.2d, v4.2s, #0
-; CHECK-NODOT-NEXT: ushll2 v16.2d, v1.4s, #0
-; CHECK-NODOT-NEXT: ushll v1.2d, v1.2s, #0
-; CHECK-NODOT-NEXT: ushll2 v3.2d, v3.4s, #0
-; CHECK-NODOT-NEXT: ushll2 v17.2d, v2.4s, #0
-; CHECK-NODOT-NEXT: ushll v2.2d, v2.2s, #0
-; CHECK-NODOT-NEXT: and v6.16b, v6.16b, v5.16b
-; CHECK-NODOT-NEXT: and v7.16b, v7.16b, v5.16b
-; CHECK-NODOT-NEXT: and v4.16b, v4.16b, v5.16b
-; CHECK-NODOT-NEXT: and v16.16b, v16.16b, v5.16b
-; CHECK-NODOT-NEXT: and v1.16b, v1.16b, v5.16b
-; CHECK-NODOT-NEXT: and v3.16b, v3.16b, v5.16b
-; CHECK-NODOT-NEXT: and v2.16b, v2.16b, v5.16b
-; CHECK-NODOT-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-NODOT-NEXT: add v1.2d, v16.2d, v4.2d
-; CHECK-NODOT-NEXT: add v4.2d, v7.2d, v6.2d
-; CHECK-NODOT-NEXT: and v6.16b, v17.16b, v5.16b
-; CHECK-NODOT-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-NODOT-NEXT: add v1.2d, v4.2d, v3.2d
-; CHECK-NODOT-NEXT: add v0.2d, v0.2d, v1.2d
-; CHECK-NODOT-NEXT: add v1.2d, v2.2d, v6.2d
-; CHECK-NODOT-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NODOT-NEXT: and v1.16b, v1.16b, v3.16b
+; CHECK-NODOT-NEXT: uaddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: uadalp v2.4s, v1.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_zext_cmp_i8tov2i64:
@@ -1729,20 +1627,10 @@ define <2 x i64> @partial_reduce_sext_cmp_i8tov2i64(<2 x i64> %acc, <16 x i8> %a
; CHECK-NODOT-LABEL: partial_reduce_sext_cmp_i8tov2i64:
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: sshll v3.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: sshll v3.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v1.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: sadalp v2.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_sext_cmp_i8tov2i64:
@@ -1802,20 +1690,10 @@ define <2 x i64> @partial_reduce_sext_cmp_i32tov2i64(<2 x i64> %acc, <4 x i32> %
define <2 x i64> @partial_reduce_sext_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %in) {
; CHECK-NODOT-LABEL: partial_reduce_sext_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: sshll v2.8h, v1.8b, #0
-; CHECK-NODOT-NEXT: sshll2 v1.8h, v1.16b, #0
-; CHECK-NODOT-NEXT: sshll v3.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v2.4s, v2.8h, #0
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: sshll v3.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v1.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
+; CHECK-NODOT-NEXT: sadalp v2.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_sext_v16i8_v2i64:
@@ -1841,20 +1719,12 @@ define <2 x i64> @partial_reduce_sext_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %in)
define <2 x i64> @partial_reduce_umull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_umull_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: umull v3.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NODOT-NEXT: umull v4.8h, v1.8b, v2.8b
; CHECK-NODOT-NEXT: umull2 v1.8h, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: ushll v4.4s, v3.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-NODOT-NEXT: ushll v2.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v2.2s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v2.4s
-; CHECK-NODOT-NEXT: uaddw v0.2d, v0.2d, v1.2s
-; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: uadalp v3.4s, v4.8h
+; CHECK-NODOT-NEXT: uadalp v3.4s, v1.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_umull_v16i8_v2i64:
@@ -1880,20 +1750,12 @@ define <2 x i64> @partial_reduce_umull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a,
define <2 x i64> @partial_reduce_smull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_smull_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: smull v3.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NODOT-NEXT: smull v4.8h, v1.8b, v2.8b
; CHECK-NODOT-NEXT: smull2 v1.8h, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: sshll v4.4s, v3.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v3.4s, v3.8h, #0
-; CHECK-NODOT-NEXT: sshll v2.4s, v1.4h, #0
-; CHECK-NODOT-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
-; CHECK-NODOT-NEXT: saddw v0.2d, v0.2d, v1.2s
-; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-NODOT-NEXT: sadalp v3.4s, v4.8h
+; CHECK-NODOT-NEXT: sadalp v3.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_smull_v16i8_v2i64:
@@ -1919,12 +1781,8 @@ define <2 x i64> @partial_reduce_smull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a,
define <2 x i64> @partial_reduce_sext_v8i16_v2i64(<2 x i64> %acc, <8 x i16> %in) {
; CHECK-COMMON-LABEL: partial_reduce_sext_v8i16_v2i64:
; CHECK-COMMON: // %bb.0:
-; CHECK-COMMON-NEXT: sshll v2.4s, v1.4h, #0
-; CHECK-COMMON-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-COMMON-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-COMMON-NEXT: saddw2 v0.2d, v0.2d, v2.4s
-; CHECK-COMMON-NEXT: saddw v0.2d, v0.2d, v1.2s
-; CHECK-COMMON-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-COMMON-NEXT: saddlp v1.4s, v1.8h
+; CHECK-COMMON-NEXT: sadalp v0.2d, v1.4s
; CHECK-COMMON-NEXT: ret
%ext = sext <8 x i16> %in to <8 x i64>
%r = call <2 x i64> @llvm.vector.partial.reduce.add.v2i64.v8i64(<2 x i64> %acc, <8 x i64> %ext)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index d0b3b19bfc44b..b9d8e58546a39 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -784,13 +784,11 @@ define <2 x i64> @four_way_i16_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
; SVE-LABEL: four_way_i16_i64_vl128:
; SVE: // %bb.0:
+; SVE-NEXT: ldr q0, [x0]
; SVE-NEXT: ldr q1, [x1]
; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: ldr q0, [x0]
-; SVE-NEXT: umull v3.4s, v2.4h, v1.4h
-; SVE-NEXT: umull2 v1.4s, v2.8h, v1.8h
-; SVE-NEXT: uadalp v0.2d, v3.4s
-; SVE-NEXT: uadalp v0.2d, v1.4s
+; SVE-NEXT: udot z0.d, z2.h, z1.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE-NEXT: ret
;
; SVE2-LABEL: four_way_i16_i64_vl128:
@@ -823,40 +821,28 @@ define <4 x i64> @four_way_i16_i64_vl128_double_width(ptr %accptr, ptr %uptr, pt
;
; NEON-LABEL: four_way_i16_i64_vl128_double_width:
; NEON: // %bb.0:
-; NEON-NEXT: ldp q0, q1, [x1]
-; NEON-NEXT: ldp q2, q3, [x2]
-; NEON-NEXT: ldp q7, q6, [x0]
-; NEON-NEXT: umull v4.4s, v3.4h, v1.4h
-; NEON-NEXT: umull v5.4s, v2.4h, v0.4h
-; NEON-NEXT: umull2 v1.4s, v3.8h, v1.8h
-; NEON-NEXT: umull2 v0.4s, v2.8h, v0.8h
-; NEON-NEXT: uaddw v7.2d, v7.2d, v5.2s
-; NEON-NEXT: uaddw v6.2d, v6.2d, v4.2s
-; NEON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
-; NEON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
-; NEON-NEXT: uaddw v2.2d, v2.2d, v0.2s
-; NEON-NEXT: uaddw v3.2d, v3.2d, v1.2s
-; NEON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
-; NEON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; NEON-NEXT: ldp q2, q5, [x2]
+; NEON-NEXT: ldp q3, q4, [x1]
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: umull v6.4s, v2.4h, v3.4h
+; NEON-NEXT: umull v7.4s, v5.4h, v4.4h
+; NEON-NEXT: umull2 v2.4s, v2.8h, v3.8h
+; NEON-NEXT: umull2 v3.4s, v5.8h, v4.8h
+; NEON-NEXT: uadalp v0.2d, v6.4s
+; NEON-NEXT: uadalp v1.2d, v7.4s
+; NEON-NEXT: uadalp v0.2d, v2.4s
+; NEON-NEXT: uadalp v1.2d, v3.4s
; NEON-NEXT: ret
;
; SVE-LABEL: four_way_i16_i64_vl128_double_width:
; SVE: // %bb.0:
-; SVE-NEXT: ldp q0, q1, [x1]
-; SVE-NEXT: ldp q2, q3, [x2]
-; SVE-NEXT: ldp q7, q6, [x0]
-; SVE-NEXT: umull v4.4s, v3.4h, v1.4h
-; SVE-NEXT: umull v5.4s, v2.4h, v0.4h
-; SVE-NEXT: umull2 v1.4s, v3.8h, v1.8h
-; SVE-NEXT: umull2 v0.4s, v2.8h, v0.8h
-; SVE-NEXT: uaddw v7.2d, v7.2d, v5.2s
-; SVE-NEXT: uaddw v6.2d, v6.2d, v4.2s
-; SVE-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
-; SVE-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
-; SVE-NEXT: uaddw v2.2d, v2.2d, v0.2s
-; SVE-NEXT: uaddw v3.2d, v3.2d, v1.2s
-; SVE-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
-; SVE-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: ldp q3, q2, [x1]
+; SVE-NEXT: ldp q5, q4, [x2]
+; SVE-NEXT: udot z0.d, z5.h, z3.h
+; SVE-NEXT: udot z1.d, z4.h, z2.h
+; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
; SVE-NEXT: ret
;
; SVE2-LABEL: four_way_i16_i64_vl128_double_width:
@@ -893,21 +879,17 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
;
; NEON-LABEL: four_way_i16_i64_vl256:
; NEON: // %bb.0:
-; NEON-NEXT: ldp q0, q1, [x1]
-; NEON-NEXT: ldp q2, q3, [x2]
-; NEON-NEXT: ldp q7, q6, [x0]
-; NEON-NEXT: umull v4.4s, v3.4h, v1.4h
-; NEON-NEXT: umull v5.4s, v2.4h, v0.4h
-; NEON-NEXT: umull2 v1.4s, v3.8h, v1.8h
-; NEON-NEXT: umull2 v0.4s, v2.8h, v0.8h
-; NEON-NEXT: uaddw v7.2d, v7.2d, v5.2s
-; NEON-NEXT: uaddw v6.2d, v6.2d, v4.2s
-; NEON-NEXT: uaddw2 v2.2d, v7.2d, v5.4s
-; NEON-NEXT: uaddw2 v3.2d, v6.2d, v4.4s
-; NEON-NEXT: uaddw v2.2d, v2.2d, v0.2s
-; NEON-NEXT: uaddw v3.2d, v3.2d, v1.2s
-; NEON-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
-; NEON-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; NEON-NEXT: ldp q2, q5, [x2]
+; NEON-NEXT: ldp q3, q4, [x1]
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: umull v6.4s, v2.4h, v3.4h
+; NEON-NEXT: umull v7.4s, v5.4h, v4.4h
+; NEON-NEXT: umull2 v2.4s, v2.8h, v3.8h
+; NEON-NEXT: umull2 v3.4s, v5.8h, v4.8h
+; NEON-NEXT: uadalp v0.2d, v6.4s
+; NEON-NEXT: uadalp v1.2d, v7.4s
+; NEON-NEXT: uadalp v0.2d, v2.4s
+; NEON-NEXT: uadalp v1.2d, v3.4s
; NEON-NEXT: ret
;
; SVE-LABEL: four_way_i16_i64_vl256:
>From c6420254f62a07cc7b4f6d3c1c1423a14ba1a00a Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 04:07:00 +0000
Subject: [PATCH 3/4] [AArch64] Select [SU]ADDLP for a zero accumulator
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 9 +++
.../neon-partial-reduce-dot-product.ll | 61 ++++++++-----------
2 files changed, 35 insertions(+), 35 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e59428f0ea33c..cca9bc37cc96a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -6079,12 +6079,21 @@ defm USQADD : SIMDTwoVectorBHSDTied<1, 0b00011, "usqadd",int_aarch64_neon_usqadd
defm XTN : SIMDMixedTwoVector<0, 0b10010, "xtn", trunc>;
// Patterns for plain partial add reductions, which lower to [SU]ADALP.
+// A zero accumulator has nothing to add to, so it lowers to [SU]ADDLP.
multiclass SelectVectorPartialReduceAdd<ValueType DstVT, ValueType SrcVT, dag immOneV> {
def : Pat<(DstVT (partial_reduce_smla DstVT:$Acc, SrcVT:$Input, (SrcVT immOneV))),
(!cast<Instruction>("SADALP" # SrcVT # "_" # DstVT) $Acc, $Input)>;
def : Pat<(DstVT (partial_reduce_umla DstVT:$Acc, SrcVT:$Input, (SrcVT immOneV))),
(!cast<Instruction>("UADALP" # SrcVT # "_" # DstVT) $Acc, $Input)>;
+
+ def : Pat<(DstVT (partial_reduce_smla (DstVT immAllZerosV), SrcVT:$Input,
+ (SrcVT immOneV))),
+ (!cast<Instruction>("SADDLP" # SrcVT # "_" # DstVT) $Input)>;
+
+ def : Pat<(DstVT (partial_reduce_umla (DstVT immAllZerosV), SrcVT:$Input,
+ (SrcVT immOneV))),
+ (!cast<Instruction>("UADDLP" # SrcVT # "_" # DstVT) $Input)>;
}
defm : SelectVectorPartialReduceAdd<v4i16, v8i8, (AArch64movi (i32 1))>;
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 2d09baf17faf3..5db4ebd82532a 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -577,12 +577,11 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: udot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
-; CHECK-NODOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NODOT-NEXT: umull v5.8h, v2.8b, v3.8b
+; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b
-; CHECK-NODOT-NEXT: uadalp v4.4s, v5.8h
-; CHECK-NODOT-NEXT: uadalp v4.4s, v2.8h
-; CHECK-NODOT-NEXT: uadalp v0.2d, v4.4s
+; CHECK-NODOT-NEXT: uaddlp v3.4s, v4.8h
+; CHECK-NODOT-NEXT: uadalp v3.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_8to64:
@@ -610,12 +609,11 @@ entry:
define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NODOT-LABEL: sdot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
-; CHECK-NODOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NODOT-NEXT: smull v5.8h, v2.8b, v3.8b
+; CHECK-NODOT-NEXT: smull v4.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: smull2 v2.8h, v2.16b, v3.16b
-; CHECK-NODOT-NEXT: sadalp v4.4s, v5.8h
-; CHECK-NODOT-NEXT: sadalp v4.4s, v2.8h
-; CHECK-NODOT-NEXT: sadalp v0.2d, v4.4s
+; CHECK-NODOT-NEXT: saddlp v3.4s, v4.8h
+; CHECK-NODOT-NEXT: sadalp v3.4s, v2.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: sdot_8to64:
@@ -909,10 +907,9 @@ define <2 x i32> @sdot_no_bin_op_narrow(<2 x i32> %acc, <8 x i8> %a){
define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: udot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
; CHECK-NODOT-NEXT: uaddlp v2.8h, v2.16b
-; CHECK-NODOT-NEXT: uadalp v3.4s, v2.8h
-; CHECK-NODOT-NEXT: uadalp v0.2d, v3.4s
+; CHECK-NODOT-NEXT: uaddlp v2.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: udot_no_bin_op_8to64:
@@ -938,10 +935,9 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-LABEL: sdot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
; CHECK-NODOT-NEXT: saddlp v2.8h, v2.16b
-; CHECK-NODOT-NEXT: sadalp v3.4s, v2.8h
-; CHECK-NODOT-NEXT: sadalp v0.2d, v3.4s
+; CHECK-NODOT-NEXT: saddlp v2.4s, v2.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: sdot_no_bin_op_8to64:
@@ -1591,11 +1587,10 @@ define <2 x i64> @partial_reduce_zext_cmp_i8tov2i64(<2 x i64> %acc, <16 x i8> %a
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: movi v3.16b, #1
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
; CHECK-NODOT-NEXT: and v1.16b, v1.16b, v3.16b
; CHECK-NODOT-NEXT: uaddlp v1.8h, v1.16b
-; CHECK-NODOT-NEXT: uadalp v2.4s, v1.8h
-; CHECK-NODOT-NEXT: uadalp v0.2d, v2.4s
+; CHECK-NODOT-NEXT: uaddlp v1.4s, v1.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v1.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_zext_cmp_i8tov2i64:
@@ -1627,10 +1622,9 @@ define <2 x i64> @partial_reduce_sext_cmp_i8tov2i64(<2 x i64> %acc, <16 x i8> %a
; CHECK-NODOT-LABEL: partial_reduce_sext_cmp_i8tov2i64:
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: cmeq v1.16b, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
-; CHECK-NODOT-NEXT: sadalp v2.4s, v1.8h
-; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
+; CHECK-NODOT-NEXT: saddlp v1.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v1.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_sext_cmp_i8tov2i64:
@@ -1690,10 +1684,9 @@ define <2 x i64> @partial_reduce_sext_cmp_i32tov2i64(<2 x i64> %acc, <4 x i32> %
define <2 x i64> @partial_reduce_sext_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %in) {
; CHECK-NODOT-LABEL: partial_reduce_sext_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: movi v2.2d, #0000000000000000
; CHECK-NODOT-NEXT: saddlp v1.8h, v1.16b
-; CHECK-NODOT-NEXT: sadalp v2.4s, v1.8h
-; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
+; CHECK-NODOT-NEXT: saddlp v1.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v1.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_sext_v16i8_v2i64:
@@ -1719,12 +1712,11 @@ define <2 x i64> @partial_reduce_sext_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %in)
define <2 x i64> @partial_reduce_umull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_umull_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NODOT-NEXT: umull v4.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: umull v3.8h, v1.8b, v2.8b
; CHECK-NODOT-NEXT: umull2 v1.8h, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: uadalp v3.4s, v4.8h
-; CHECK-NODOT-NEXT: uadalp v3.4s, v1.8h
-; CHECK-NODOT-NEXT: uadalp v0.2d, v3.4s
+; CHECK-NODOT-NEXT: uaddlp v2.4s, v3.8h
+; CHECK-NODOT-NEXT: uadalp v2.4s, v1.8h
+; CHECK-NODOT-NEXT: uadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_umull_v16i8_v2i64:
@@ -1750,12 +1742,11 @@ define <2 x i64> @partial_reduce_umull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a,
define <2 x i64> @partial_reduce_smull_v16i8_v2i64(<2 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-LABEL: partial_reduce_smull_v16i8_v2i64:
; CHECK-NODOT: // %bb.0:
-; CHECK-NODOT-NEXT: movi v3.2d, #0000000000000000
-; CHECK-NODOT-NEXT: smull v4.8h, v1.8b, v2.8b
+; CHECK-NODOT-NEXT: smull v3.8h, v1.8b, v2.8b
; CHECK-NODOT-NEXT: smull2 v1.8h, v1.16b, v2.16b
-; CHECK-NODOT-NEXT: sadalp v3.4s, v4.8h
-; CHECK-NODOT-NEXT: sadalp v3.4s, v1.8h
-; CHECK-NODOT-NEXT: sadalp v0.2d, v3.4s
+; CHECK-NODOT-NEXT: saddlp v2.4s, v3.8h
+; CHECK-NODOT-NEXT: sadalp v2.4s, v1.8h
+; CHECK-NODOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-NODOT-NEXT: ret
;
; CHECK-DOT-LABEL: partial_reduce_smull_v16i8_v2i64:
>From 13c0ea298ac83788003a46bd9187d547c0fb74e5 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 04:41:46 +0000
Subject: [PATCH 4/4] [AArch64] Keep an extend feeding a partial reduction out
of TBL
---
.../Target/AArch64/AArch64ISelLowering.cpp | 7 ++
.../neon-partial-reduce-dot-product.ll | 93 +++++++++++--------
2 files changed, 62 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 68bd6cb1a78de..966049c48ba51 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -19199,6 +19199,13 @@ bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(
m_Intrinsic<Intrinsic::vector_partial_reduce_add>(
m_Value(), m_Specific(I))))
return true;
+ // The extend can also reach a partial reduction through a mul
+ // of two extends, which folds into the reduction.
+ if (match(SingleUser, m_c_Mul(m_Specific(I), m_ZExt(m_Value()))) &&
+ SingleUser->hasOneUse() &&
+ match(SingleUser->user_back(),
+ m_Intrinsic<Intrinsic::vector_partial_reduce_add>()))
+ return true;
return false;
}))
return false;
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 5db4ebd82532a..d02173ee86e6f 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1796,44 +1796,61 @@ define <2 x i64> @partial_reduce_smull_v8i16_v2i64(<2 x i64> %acc, <8 x i16> %a,
}
define <2 x i64> @udot_in_loop_8to64(ptr %p1, ptr %p2){
-; CHECK-COMMON-LABEL: udot_in_loop_8to64:
-; CHECK-COMMON: // %bb.0: // %entry
-; CHECK-COMMON-NEXT: adrp x8, .LCPI54_0
-; CHECK-COMMON-NEXT: movi v1.2d, #0000000000000000
-; CHECK-COMMON-NEXT: adrp x9, .LCPI54_2
-; CHECK-COMMON-NEXT: ldr q2, [x8, :lo12:.LCPI54_0]
-; CHECK-COMMON-NEXT: adrp x8, .LCPI54_1
-; CHECK-COMMON-NEXT: adrp x10, .LCPI54_3
-; CHECK-COMMON-NEXT: ldr q3, [x8, :lo12:.LCPI54_1]
-; CHECK-COMMON-NEXT: ldr q4, [x9, :lo12:.LCPI54_2]
-; CHECK-COMMON-NEXT: ldr q5, [x10, :lo12:.LCPI54_3]
-; CHECK-COMMON-NEXT: mov x8, xzr
-; CHECK-COMMON-NEXT: .LBB54_1: // %vector.body
-; CHECK-COMMON-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-COMMON-NEXT: ldr q6, [x0, x8]
-; CHECK-COMMON-NEXT: ldr q7, [x1, x8]
-; CHECK-COMMON-NEXT: mov v0.16b, v1.16b
-; CHECK-COMMON-NEXT: add x8, x8, #16
-; CHECK-COMMON-NEXT: tbl v16.16b, { v6.16b }, v2.16b
-; CHECK-COMMON-NEXT: tbl v17.16b, { v7.16b }, v2.16b
-; CHECK-COMMON-NEXT: tbl v18.16b, { v6.16b }, v3.16b
-; CHECK-COMMON-NEXT: tbl v19.16b, { v7.16b }, v3.16b
-; CHECK-COMMON-NEXT: cmp x8, #16
-; CHECK-COMMON-NEXT: umlal v1.2d, v16.2s, v17.2s
-; CHECK-COMMON-NEXT: umlal2 v1.2d, v16.4s, v17.4s
-; CHECK-COMMON-NEXT: tbl v16.16b, { v6.16b }, v4.16b
-; CHECK-COMMON-NEXT: tbl v17.16b, { v7.16b }, v4.16b
-; CHECK-COMMON-NEXT: tbl v6.16b, { v6.16b }, v5.16b
-; CHECK-COMMON-NEXT: tbl v7.16b, { v7.16b }, v5.16b
-; CHECK-COMMON-NEXT: umlal v1.2d, v18.2s, v19.2s
-; CHECK-COMMON-NEXT: umlal2 v1.2d, v18.4s, v19.4s
-; CHECK-COMMON-NEXT: umlal v1.2d, v16.2s, v17.2s
-; CHECK-COMMON-NEXT: umlal2 v1.2d, v16.4s, v17.4s
-; CHECK-COMMON-NEXT: umlal v1.2d, v6.2s, v7.2s
-; CHECK-COMMON-NEXT: umlal2 v1.2d, v6.4s, v7.4s
-; CHECK-COMMON-NEXT: b.ne .LBB54_1
-; CHECK-COMMON-NEXT: // %bb.2: // %end
-; CHECK-COMMON-NEXT: ret
+; CHECK-NODOT-LABEL: udot_in_loop_8to64:
+; CHECK-NODOT: // %bb.0: // %entry
+; CHECK-NODOT-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NODOT-NEXT: mov x8, xzr
+; CHECK-NODOT-NEXT: .LBB54_1: // %vector.body
+; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NODOT-NEXT: ldr q0, [x0, x8]
+; CHECK-NODOT-NEXT: ldr q2, [x1, x8]
+; CHECK-NODOT-NEXT: add x8, x8, #16
+; CHECK-NODOT-NEXT: cmp x8, #16
+; CHECK-NODOT-NEXT: umull v3.8h, v0.8b, v2.8b
+; CHECK-NODOT-NEXT: umull2 v0.8h, v0.16b, v2.16b
+; CHECK-NODOT-NEXT: uaddlp v2.4s, v3.8h
+; CHECK-NODOT-NEXT: uadalp v2.4s, v0.8h
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT: uadalp v1.2d, v2.4s
+; CHECK-NODOT-NEXT: b.ne .LBB54_1
+; CHECK-NODOT-NEXT: // %bb.2: // %end
+; CHECK-NODOT-NEXT: ret
+;
+; CHECK-DOT-LABEL: udot_in_loop_8to64:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-NEXT: mov x8, xzr
+; CHECK-DOT-NEXT: .LBB54_1: // %vector.body
+; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: ldr q0, [x0, x8]
+; CHECK-DOT-NEXT: ldr q3, [x1, x8]
+; CHECK-DOT-NEXT: add x8, x8, #16
+; CHECK-DOT-NEXT: cmp x8, #16
+; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v3.16b
+; CHECK-DOT-NEXT: mov v0.16b, v1.16b
+; CHECK-DOT-NEXT: uadalp v1.2d, v2.4s
+; CHECK-DOT-NEXT: b.ne .LBB54_1
+; CHECK-DOT-NEXT: // %bb.2: // %end
+; CHECK-DOT-NEXT: ret
+;
+; CHECK-DOT-I8MM-LABEL: udot_in_loop_8to64:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: mov x8, xzr
+; CHECK-DOT-I8MM-NEXT: .LBB54_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: ldr q0, [x0, x8]
+; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-DOT-I8MM-NEXT: add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT: cmp x8, #16
+; CHECK-DOT-I8MM-NEXT: udot v2.4s, v0.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT: uadalp v1.2d, v2.4s
+; CHECK-DOT-I8MM-NEXT: b.ne .LBB54_1
+; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT: ret
entry:
br label %vector.body
More information about the llvm-commits
mailing list