[llvm] [AArch64] Use [SU]ADALP for partial (nx)v4i32 -> (nx)v2i64 add reduce. (PR #213915)
Ricardo Jesus via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 04:55:05 PDT 2026
https://github.com/rj-jesus created https://github.com/llvm/llvm-project/pull/213915
The previously used AArch64ISD::[SU]ADDW[BT] became unused after this change, so I've removed them. Please let me know if they should be kept instead.
>From 0d5835f3d6c8ceeb3f7eff146187b48b327fdc25 Mon Sep 17 00:00:00 2001
From: Ricardo Jesus <rjj at nvidia.com>
Date: Tue, 4 Aug 2026 01:44:24 -0700
Subject: [PATCH] [AArch64] Use [SU]ADALP for partial (nx)v4i32 -> (nx)v2i64
dot widening.
The previously used AArch64ISD::[SU]ADDW[BT] became unused due to this
change, so I've removed them, but please let me know if it's preferable
they are kept.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 32 +--
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 16 +-
.../AArch64/AArch64SelectionDAGInfo.cpp | 19 --
.../neon-partial-reduce-dot-product.ll | 42 +--
.../sve-fixed-length-partial-reduce.ll | 257 ++++++++----------
.../AArch64/sve-partial-reduce-dot-product.ll | 64 ++---
6 files changed, 183 insertions(+), 247 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 629f4e98322c2..81e270c7befe0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -25251,18 +25251,6 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bic_u:
return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0),
N->getOperand(2), N->getOperand(3));
- case Intrinsic::aarch64_sve_saddwb:
- return DAG.getNode(AArch64ISD::SADDWB, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2));
- case Intrinsic::aarch64_sve_saddwt:
- return DAG.getNode(AArch64ISD::SADDWT, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2));
- case Intrinsic::aarch64_sve_uaddwb:
- return DAG.getNode(AArch64ISD::UADDWB, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2));
- case Intrinsic::aarch64_sve_uaddwt:
- return DAG.getNode(AArch64ISD::UADDWT, SDLoc(N), N->getValueType(0),
- N->getOperand(1), N->getOperand(2));
case Intrinsic::aarch64_sve_eor_u:
return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2),
N->getOperand(3));
@@ -34755,11 +34743,23 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
+ // Attempt to fold v4i32 -> v2i64 via Neon *ADALP.
+ if (Subtarget->isNeonAvailable() && OrigResultVT == MVT::v2i64) {
+ unsigned Opc = IsUnsigned ? AArch64ISD::UADDLP : AArch64ISD::SADDLP;
+ if (ConvertToScalable)
+ DotNode = convertFromScalableVector(DAG, MVT::v4i32, DotNode);
+ SDValue Res = DAG.getNode(Opc, DL, OrigResultVT, DotNode);
+ return DAG.getNode(ISD::ADD, DL, OrigResultVT, OrigAcc, Res);
+ }
+
+ // Otherwise use SVE2 *ADALP.
if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
- unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
- unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
- SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
- SDValue Res = DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
+ unsigned IID = IsUnsigned ? Intrinsic::aarch64_sve_uadalp
+ : Intrinsic::aarch64_sve_sadalp;
+ SDValue Pg = getPredicateForVector(DAG, DL, ResultVT);
+ SDValue Res =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResultVT,
+ DAG.getConstant(IID, DL, MVT::i64), Pg, Acc, DotNode);
return ConvertToScalable ? convertFromScalableVector(DAG, OrigResultVT, Res)
: Res;
}
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 55a3db86940be..ea513f168e885 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -520,14 +520,6 @@ def SDT_AArch64Arith_Unpred : SDTypeProfile<1, 2, [
// Unpredicated vector instructions
def AArch64bic_node : SDNode<"AArch64ISD::BIC", SDT_AArch64Arith_Unpred>;
-def SDT_AArch64addw : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>]>;
-
-// Wide adds
-def AArch64saddwt : SDNode<"AArch64ISD::SADDWT", SDT_AArch64addw>;
-def AArch64saddwb : SDNode<"AArch64ISD::SADDWB", SDT_AArch64addw>;
-def AArch64uaddwt : SDNode<"AArch64ISD::UADDWT", SDT_AArch64addw>;
-def AArch64uaddwb : SDNode<"AArch64ISD::UADDWB", SDT_AArch64addw>;
-
def AArch64bic : PatFrags<(ops node:$op1, node:$op2),
[(and node:$op1, (xor node:$op2, (splat_vector (i32 -1)))),
(and node:$op1, (xor node:$op2, (splat_vector (i64 -1)))),
@@ -4141,10 +4133,10 @@ let Predicates = [HasSVE2_or_SME] in {
defm UABDLT_ZZZ : sve2_wide_int_arith_long<0b01111, "uabdlt", int_aarch64_sve_uabdlt>;
// SVE2 integer add/subtract wide
- defm SADDWB_ZZZ : sve2_wide_int_arith_wide<0b000, "saddwb", AArch64saddwb>;
- defm SADDWT_ZZZ : sve2_wide_int_arith_wide<0b001, "saddwt", AArch64saddwt>;
- defm UADDWB_ZZZ : sve2_wide_int_arith_wide<0b010, "uaddwb", AArch64uaddwb>;
- defm UADDWT_ZZZ : sve2_wide_int_arith_wide<0b011, "uaddwt", AArch64uaddwt>;
+ defm SADDWB_ZZZ : sve2_wide_int_arith_wide<0b000, "saddwb", int_aarch64_sve_saddwb>;
+ defm SADDWT_ZZZ : sve2_wide_int_arith_wide<0b001, "saddwt", int_aarch64_sve_saddwt>;
+ defm UADDWB_ZZZ : sve2_wide_int_arith_wide<0b010, "uaddwb", int_aarch64_sve_uaddwb>;
+ defm UADDWT_ZZZ : sve2_wide_int_arith_wide<0b011, "uaddwt", int_aarch64_sve_uaddwt>;
defm SSUBWB_ZZZ : sve2_wide_int_arith_wide<0b100, "ssubwb", int_aarch64_sve_ssubwb>;
defm SSUBWT_ZZZ : sve2_wide_int_arith_wide<0b101, "ssubwt", int_aarch64_sve_ssubwt>;
defm USUBWB_ZZZ : sve2_wide_int_arith_wide<0b110, "usubwb", int_aarch64_sve_usubwb>;
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 286e5638fc088..7eac36af4ab46 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -52,25 +52,6 @@ void AArch64SelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
"Expected the general-predicate and mask to have matching types");
break;
- case AArch64ISD::SADDWT:
- case AArch64ISD::SADDWB:
- case AArch64ISD::UADDWT:
- case AArch64ISD::UADDWB: {
- EVT VT = N->getValueType(0);
- EVT Op0VT = N->getOperand(0).getValueType();
- EVT Op1VT = N->getOperand(1).getValueType();
- assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&
- VT.isInteger() && Op0VT.isInteger() && Op1VT.isInteger() &&
- "Expected integer vectors!");
- assert(VT == Op0VT &&
- "Expected result and first input to have the same type!");
- assert(Op0VT.getSizeInBits() == Op1VT.getSizeInBits() &&
- "Expected vectors of equal size!");
- assert(Op0VT.getVectorElementCount() * 2 == Op1VT.getVectorElementCount() &&
- "Expected result vector and first input vector to have half the "
- "lanes of the second input vector!");
- break;
- }
case AArch64ISD::SUNPKLO:
case AArch64ISD::SUNPKHI:
case AArch64ISD::UUNPKLO:
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index b5801f8f48057..d09cf29768158 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -617,16 +617,14 @@ define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-DOT: // %bb.0: // %entry
; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-NEXT: uadalp v0.2d, v4.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: udot_8to64:
; CHECK-DOT-I8MM: // %bb.0: // %entry
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: uadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
entry:
%a.wide = zext <16 x i8> %a to <16 x i64>
@@ -660,16 +658,14 @@ define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-DOT: // %bb.0: // %entry
; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: sdot_8to64:
; CHECK-DOT-I8MM: // %bb.0: // %entry
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
entry:
%a.wide = sext <16 x i8> %a to <16 x i64>
@@ -714,16 +710,14 @@ define <4 x i64> @usdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-DOT-NEXT: udot v6.4s, v4.16b, v2.16b
; CHECK-DOT-NEXT: udot v5.4s, v3.16b, v2.16b
; CHECK-DOT-NEXT: sub v2.4s, v5.4s, v6.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
+; CHECK-DOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: usdot_8to64:
; CHECK-DOT-I8MM: // %bb.0: // %entry
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
entry:
%a.wide = zext <16 x i8> %a to <16 x i64>
@@ -768,16 +762,14 @@ define <4 x i64> @sudot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-DOT-NEXT: udot v6.4s, v4.16b, v3.16b
; CHECK-DOT-NEXT: udot v5.4s, v2.16b, v3.16b
; CHECK-DOT-NEXT: sub v2.4s, v5.4s, v6.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v2.2s
-; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v2.4s
+; CHECK-DOT-NEXT: sadalp v0.2d, v2.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: sudot_8to64:
; CHECK-DOT-I8MM: // %bb.0: // %entry
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v3.16b, v2.16b
-; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
entry:
%a.wide = sext <16 x i8> %a to <16 x i64>
@@ -1000,8 +992,7 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-DOT-NEXT: movi v3.16b, #1
; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-NEXT: uadalp v0.2d, v4.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_8to64:
@@ -1009,8 +1000,7 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-DOT-I8MM-NEXT: movi v3.16b, #1
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: uadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
%a.wide = zext <16 x i8> %a to <16 x i64>
%partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)
@@ -1041,8 +1031,7 @@ define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-DOT-NEXT: movi v3.16b, #1
; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: sdot_no_bin_op_8to64:
@@ -1050,8 +1039,7 @@ define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-DOT-I8MM-NEXT: movi v3.16b, #1
; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v4.4s
; CHECK-DOT-I8MM-NEXT: ret
%a.wide = sext <16 x i8> %a to <16 x i64>
%partial.reduce = tail call <4 x i64> @llvm.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)
@@ -1574,16 +1562,14 @@ define <2 x i64> @usdot_v16i8tov2i64(<2 x i64> %acc, <16 x i8> %u, <16 x i8> %s)
; CHECK-DOT-NEXT: udot v5.4s, v3.16b, v1.16b
; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v1.16b
; CHECK-DOT-NEXT: sub v1.4s, v4.4s, v5.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v1.2s
-; CHECK-DOT-NEXT: saddw2 v0.2d, v0.2d, v1.4s
+; CHECK-DOT-NEXT: sadalp v0.2d, v1.4s
; CHECK-DOT-NEXT: ret
;
; CHECK-DOT-I8MM-LABEL: usdot_v16i8tov2i64:
; CHECK-DOT-I8MM: // %bb.0: // %entry
; CHECK-DOT-I8MM-NEXT: movi v3.2d, #0000000000000000
; CHECK-DOT-I8MM-NEXT: usdot v3.4s, v1.16b, v2.16b
-; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v3.2s
-; CHECK-DOT-I8MM-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-DOT-I8MM-NEXT: sadalp v0.2d, v3.4s
; CHECK-DOT-I8MM-NEXT: ret
entry:
%u.wide = zext <16 x i8> %u to <16 x i64>
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index 5d2787610f4ad..b757939dcfa26 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -632,47 +632,43 @@ define <4 x i32> @four_way_i8_i32_vl128_sudot(ptr %accptr, ptr %uptr, ptr %sptr)
define <2 x i64> @four_way_i8_i64_vl128_usdot(ptr %accptr, ptr %uptr, ptr %sptr) {
; NEON-LABEL: four_way_i8_i64_vl128_usdot:
; NEON: // %bb.0:
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: movi v1.2d, #0000000000000000
+; NEON-NEXT: ldr q0, [x1]
; NEON-NEXT: ldr q2, [x2]
-; NEON-NEXT: usdot v0.4s, v1.16b, v2.16b
-; NEON-NEXT: ldr q1, [x0]
-; NEON-NEXT: saddw v1.2d, v1.2d, v0.2s
-; NEON-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: usdot v1.4s, v0.16b, v2.16b
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: sadalp v0.2d, v1.4s
; NEON-NEXT: ret
;
; SVE-LABEL: four_way_i8_i64_vl128_usdot:
; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: movi v1.2d, #0000000000000000
+; SVE-NEXT: ldr q0, [x1]
; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: usdot z0.s, z1.b, z2.b
-; SVE-NEXT: ldr q1, [x0]
-; SVE-NEXT: saddw v1.2d, v1.2d, v0.2s
-; SVE-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: usdot z1.s, z0.b, z2.b
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: sadalp v0.2d, v1.4s
; SVE-NEXT: ret
;
; SVE2-LABEL: four_way_i8_i64_vl128_usdot:
; SVE2: // %bb.0:
-; SVE2-NEXT: movi v0.2d, #0000000000000000
-; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: movi v1.2d, #0000000000000000
+; SVE2-NEXT: ldr q0, [x1]
; SVE2-NEXT: ldr q2, [x2]
-; SVE2-NEXT: usdot z0.s, z1.b, z2.b
-; SVE2-NEXT: ldr q1, [x0]
-; SVE2-NEXT: saddwb z1.d, z1.d, z0.s
-; SVE2-NEXT: saddwt z0.d, z1.d, z0.s
-; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: usdot z1.s, z0.b, z2.b
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: sadalp v0.2d, v1.4s
; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_usdot:
; SME: // %bb.0:
-; SME-NEXT: mov z0.s, #0 // =0x0
-; SME-NEXT: ldr q1, [x1]
+; SME-NEXT: mov z1.s, #0 // =0x0
+; SME-NEXT: ldr q0, [x1]
; SME-NEXT: ldr q2, [x2]
-; SME-NEXT: usdot z0.s, z1.b, z2.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: saddwb z1.d, z1.d, z0.s
-; SME-NEXT: saddwt z0.d, z1.d, z0.s
+; SME-NEXT: ptrue p0.d
+; SME-NEXT: usdot z1.s, z0.b, z2.b
+; SME-NEXT: ldr q0, [x0]
+; SME-NEXT: sadalp z0.d, p0/m, z1.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -1119,47 +1115,43 @@ define <2 x i64> @eight_way_i8_i64_vl128(ptr %accptr, ptr %uptr, ptr %sptr) {
;
; NEON-LABEL: eight_way_i8_i64_vl128:
; NEON: // %bb.0:
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: movi v1.2d, #0000000000000000
+; NEON-NEXT: ldr q0, [x1]
; NEON-NEXT: ldr q2, [x2]
-; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
-; NEON-NEXT: ldr q1, [x0]
-; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: udot v1.4s, v2.16b, v0.16b
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: uadalp v0.2d, v1.4s
; NEON-NEXT: ret
;
; SVE-LABEL: eight_way_i8_i64_vl128:
; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: movi v1.2d, #0000000000000000
+; SVE-NEXT: ldr q0, [x1]
; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: udot z0.s, z2.b, z1.b
-; SVE-NEXT: ldr q1, [x0]
-; SVE-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: udot z1.s, z2.b, z0.b
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: uadalp v0.2d, v1.4s
; SVE-NEXT: ret
;
; SVE2-LABEL: eight_way_i8_i64_vl128:
; SVE2: // %bb.0:
-; SVE2-NEXT: movi v0.2d, #0000000000000000
-; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: movi v1.2d, #0000000000000000
+; SVE2-NEXT: ldr q0, [x1]
; SVE2-NEXT: ldr q2, [x2]
-; SVE2-NEXT: udot z0.s, z2.b, z1.b
-; SVE2-NEXT: ldr q1, [x0]
-; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
-; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
-; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: udot z1.s, z2.b, z0.b
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: uadalp v0.2d, v1.4s
; SVE2-NEXT: ret
;
; SME-LABEL: eight_way_i8_i64_vl128:
; SME: // %bb.0:
-; SME-NEXT: mov z0.s, #0 // =0x0
-; SME-NEXT: ldr q1, [x1]
+; SME-NEXT: mov z1.s, #0 // =0x0
+; SME-NEXT: ldr q0, [x1]
; SME-NEXT: ldr q2, [x2]
-; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: uaddwb z1.d, z1.d, z0.s
-; SME-NEXT: uaddwt z0.d, z1.d, z0.s
+; SME-NEXT: ptrue p0.d
+; SME-NEXT: udot z1.s, z2.b, z0.b
+; SME-NEXT: ldr q0, [x0]
+; SME-NEXT: uadalp z0.d, p0/m, z1.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -1180,47 +1172,43 @@ define <2 x i64> @eight_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
;
; NEON-LABEL: eight_way_i8_i64_vl256:
; NEON: // %bb.0:
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldr q1, [x1]
+; NEON-NEXT: movi v1.2d, #0000000000000000
+; NEON-NEXT: ldr q0, [x1]
; NEON-NEXT: ldr q2, [x2]
-; NEON-NEXT: udot v0.4s, v2.16b, v1.16b
-; NEON-NEXT: ldr q1, [x0]
-; NEON-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; NEON-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; NEON-NEXT: udot v1.4s, v2.16b, v0.16b
+; NEON-NEXT: ldr q0, [x0]
+; NEON-NEXT: uadalp v0.2d, v1.4s
; NEON-NEXT: ret
;
; SVE-LABEL: eight_way_i8_i64_vl256:
; SVE: // %bb.0:
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldr q1, [x1]
+; SVE-NEXT: movi v1.2d, #0000000000000000
+; SVE-NEXT: ldr q0, [x1]
; SVE-NEXT: ldr q2, [x2]
-; SVE-NEXT: udot z0.s, z2.b, z1.b
-; SVE-NEXT: ldr q1, [x0]
-; SVE-NEXT: uaddw v1.2d, v1.2d, v0.2s
-; SVE-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; SVE-NEXT: udot z1.s, z2.b, z0.b
+; SVE-NEXT: ldr q0, [x0]
+; SVE-NEXT: uadalp v0.2d, v1.4s
; SVE-NEXT: ret
;
; SVE2-LABEL: eight_way_i8_i64_vl256:
; SVE2: // %bb.0:
-; SVE2-NEXT: movi v0.2d, #0000000000000000
-; SVE2-NEXT: ldr q1, [x1]
+; SVE2-NEXT: movi v1.2d, #0000000000000000
+; SVE2-NEXT: ldr q0, [x1]
; SVE2-NEXT: ldr q2, [x2]
-; SVE2-NEXT: udot z0.s, z2.b, z1.b
-; SVE2-NEXT: ldr q1, [x0]
-; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
-; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
-; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
+; SVE2-NEXT: udot z1.s, z2.b, z0.b
+; SVE2-NEXT: ldr q0, [x0]
+; SVE2-NEXT: uadalp v0.2d, v1.4s
; SVE2-NEXT: ret
;
; SME-LABEL: eight_way_i8_i64_vl256:
; SME: // %bb.0:
-; SME-NEXT: mov z0.s, #0 // =0x0
-; SME-NEXT: ldr q1, [x1]
+; SME-NEXT: mov z1.s, #0 // =0x0
+; SME-NEXT: ldr q0, [x1]
; SME-NEXT: ldr q2, [x2]
-; SME-NEXT: udot z0.s, z2.b, z1.b
-; SME-NEXT: ldr q1, [x0]
-; SME-NEXT: uaddwb z1.d, z1.d, z0.s
-; SME-NEXT: uaddwt z0.d, z1.d, z0.s
+; SME-NEXT: ptrue p0.d
+; SME-NEXT: udot z1.s, z2.b, z0.b
+; SME-NEXT: ldr q0, [x0]
+; SME-NEXT: uadalp z0.d, p0/m, z1.s
; SME-NEXT: ret
%acc = load <2 x i64>, ptr %accptr
%u = load <16 x i8>, ptr %uptr
@@ -1236,64 +1224,55 @@ define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr
;
; NEON-LABEL: four_way_i8_i64_vl128_double_width:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.2d, #0000000000000000
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldp q3, q2, [x1]
-; NEON-NEXT: ldp q5, q4, [x2]
-; NEON-NEXT: udot v0.4s, v5.16b, v3.16b
-; NEON-NEXT: udot v1.4s, v4.16b, v2.16b
-; NEON-NEXT: ldp q3, q2, [x0]
-; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s
-; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s
-; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
-; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; NEON-NEXT: movi v2.2d, #0000000000000000
+; NEON-NEXT: movi v3.2d, #0000000000000000
+; NEON-NEXT: ldp q0, q5, [x2]
+; NEON-NEXT: ldp q1, q4, [x1]
+; NEON-NEXT: udot v2.4s, v0.16b, v1.16b
+; NEON-NEXT: udot v3.4s, v5.16b, v4.16b
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: uadalp v0.2d, v2.4s
+; NEON-NEXT: uadalp v1.2d, v3.4s
; NEON-NEXT: ret
;
; SVE-LABEL: four_way_i8_i64_vl128_double_width:
; SVE: // %bb.0:
-; SVE-NEXT: movi v1.2d, #0000000000000000
-; SVE-NEXT: movi v0.2d, #0000000000000000
-; SVE-NEXT: ldp q3, q2, [x1]
-; SVE-NEXT: ldp q5, q4, [x2]
-; SVE-NEXT: udot z0.s, z5.b, z3.b
-; SVE-NEXT: udot z1.s, z4.b, z2.b
-; SVE-NEXT: ldp q3, q2, [x0]
-; SVE-NEXT: uaddw v3.2d, v3.2d, v0.2s
-; SVE-NEXT: uaddw v2.2d, v2.2d, v1.2s
-; SVE-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
-; SVE-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; SVE-NEXT: movi v2.2d, #0000000000000000
+; SVE-NEXT: movi v3.2d, #0000000000000000
+; SVE-NEXT: ldp q0, q5, [x2]
+; SVE-NEXT: ldp q1, q4, [x1]
+; SVE-NEXT: udot z2.s, z0.b, z1.b
+; SVE-NEXT: udot z3.s, z5.b, z4.b
+; SVE-NEXT: ldp q0, q1, [x0]
+; SVE-NEXT: uadalp v0.2d, v2.4s
+; SVE-NEXT: uadalp v1.2d, v3.4s
; SVE-NEXT: ret
;
; SVE2-LABEL: four_way_i8_i64_vl128_double_width:
; SVE2: // %bb.0:
-; SVE2-NEXT: movi v1.2d, #0000000000000000
-; SVE2-NEXT: movi v0.2d, #0000000000000000
-; SVE2-NEXT: ldp q3, q2, [x1]
-; SVE2-NEXT: ldp q5, q4, [x2]
-; SVE2-NEXT: udot z0.s, z5.b, z3.b
-; SVE2-NEXT: udot z1.s, z4.b, z2.b
-; SVE2-NEXT: ldp q3, q2, [x0]
-; SVE2-NEXT: uaddwb z3.d, z3.d, z0.s
-; SVE2-NEXT: uaddwb z2.d, z2.d, z1.s
-; SVE2-NEXT: uaddwt z0.d, z3.d, z0.s
-; SVE2-NEXT: uaddwt z1.d, z2.d, z1.s
-; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
-; SVE2-NEXT: // kill: def $q1 killed $q1 killed $z1
+; SVE2-NEXT: movi v2.2d, #0000000000000000
+; SVE2-NEXT: movi v3.2d, #0000000000000000
+; SVE2-NEXT: ldp q0, q5, [x2]
+; SVE2-NEXT: ldp q1, q4, [x1]
+; SVE2-NEXT: udot z2.s, z0.b, z1.b
+; SVE2-NEXT: udot z3.s, z5.b, z4.b
+; SVE2-NEXT: ldp q0, q1, [x0]
+; SVE2-NEXT: uadalp v0.2d, v2.4s
+; SVE2-NEXT: uadalp v1.2d, v3.4s
; SVE2-NEXT: ret
;
; SME-LABEL: four_way_i8_i64_vl128_double_width:
; SME: // %bb.0:
-; SME-NEXT: mov z1.s, #0 // =0x0
-; SME-NEXT: mov z0.s, #0 // =0x0
-; SME-NEXT: ldp q3, q2, [x1]
-; SME-NEXT: ldp q5, q4, [x2]
-; SME-NEXT: udot z0.s, z5.b, z3.b
-; SME-NEXT: udot z1.s, z4.b, z2.b
-; SME-NEXT: ldp q3, q2, [x0]
-; SME-NEXT: uaddwb z3.d, z3.d, z0.s
-; SME-NEXT: uaddwb z2.d, z2.d, z1.s
-; SME-NEXT: uaddwt z0.d, z3.d, z0.s
-; SME-NEXT: uaddwt z1.d, z2.d, z1.s
+; SME-NEXT: mov z2.s, #0 // =0x0
+; SME-NEXT: mov z3.s, #0 // =0x0
+; SME-NEXT: ptrue p0.d
+; SME-NEXT: ldp q0, q5, [x2]
+; SME-NEXT: ldp q1, q4, [x1]
+; SME-NEXT: udot z2.s, z0.b, z1.b
+; SME-NEXT: udot z3.s, z5.b, z4.b
+; SME-NEXT: ldp q0, q1, [x0]
+; SME-NEXT: uadalp z0.d, p0/m, z2.s
+; SME-NEXT: uadalp z1.d, p0/m, z3.s
; SME-NEXT: ret
%acc = load <4 x i64>, ptr %accptr
%u = load <32 x i8>, ptr %uptr
@@ -1314,17 +1293,15 @@ define <4 x i64> @four_way_i8_i64_vl128_double_width(ptr %accptr, ptr %uptr, ptr
define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscale_range(2,2) {
; NEON-LABEL: four_way_i8_i64_vl256:
; NEON: // %bb.0:
-; NEON-NEXT: movi v1.2d, #0000000000000000
-; NEON-NEXT: movi v0.2d, #0000000000000000
-; NEON-NEXT: ldp q3, q2, [x1]
-; NEON-NEXT: ldp q5, q4, [x2]
-; NEON-NEXT: udot v0.4s, v5.16b, v3.16b
-; NEON-NEXT: udot v1.4s, v4.16b, v2.16b
-; NEON-NEXT: ldp q3, q2, [x0]
-; NEON-NEXT: uaddw v3.2d, v3.2d, v0.2s
-; NEON-NEXT: uaddw v2.2d, v2.2d, v1.2s
-; NEON-NEXT: uaddw2 v0.2d, v3.2d, v0.4s
-; NEON-NEXT: uaddw2 v1.2d, v2.2d, v1.4s
+; NEON-NEXT: movi v2.2d, #0000000000000000
+; NEON-NEXT: movi v3.2d, #0000000000000000
+; NEON-NEXT: ldp q0, q5, [x2]
+; NEON-NEXT: ldp q1, q4, [x1]
+; NEON-NEXT: udot v2.4s, v0.16b, v1.16b
+; NEON-NEXT: udot v3.4s, v5.16b, v4.16b
+; NEON-NEXT: ldp q0, q1, [x0]
+; NEON-NEXT: uadalp v0.2d, v2.4s
+; NEON-NEXT: uadalp v1.2d, v3.4s
; NEON-NEXT: ret
;
; SVE-LABEL: four_way_i8_i64_vl256:
@@ -1347,13 +1324,13 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
;
; SVE2-LABEL: four_way_i8_i64_vl256:
; SVE2: // %bb.0:
-; SVE2-NEXT: movi v0.2d, #0000000000000000
-; SVE2-NEXT: ldr z1, [x1]
+; SVE2-NEXT: movi v1.2d, #0000000000000000
+; SVE2-NEXT: ldr z0, [x1]
; SVE2-NEXT: ldr z2, [x2]
-; SVE2-NEXT: udot z0.s, z2.b, z1.b
-; SVE2-NEXT: ldr z1, [x0]
-; SVE2-NEXT: uaddwb z1.d, z1.d, z0.s
-; SVE2-NEXT: uaddwt z0.d, z1.d, z0.s
+; SVE2-NEXT: ptrue p0.d
+; SVE2-NEXT: udot z1.s, z2.b, z0.b
+; SVE2-NEXT: ldr z0, [x0]
+; SVE2-NEXT: uadalp z0.d, p0/m, z1.s
; SVE2-NEXT: movprfx z1, z0
; SVE2-NEXT: ext z1.b, z1.b, z0.b, #16
; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
@@ -1365,10 +1342,10 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
; SME-NEXT: ldr z0, [x1]
; SME-NEXT: ldr z1, [x2]
; SME-NEXT: mov z2.s, #0 // =0x0
+; SME-NEXT: ptrue p0.d
; SME-NEXT: udot z2.s, z1.b, z0.b
; SME-NEXT: ldr z0, [x0]
-; SME-NEXT: uaddwb z0.d, z0.d, z2.s
-; SME-NEXT: uaddwt z0.d, z0.d, z2.s
+; SME-NEXT: uadalp z0.d, p0/m, z2.s
; SME-NEXT: movprfx z1, z0
; SME-NEXT: ext z1.b, z1.b, z0.b, #16
; SME-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
index 836a22b3d69f6..db6ead120c7de 100644
--- a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
@@ -177,25 +177,25 @@ define <vscale x 4 x i64> @udot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8
; CHECK-SVE2-LABEL: udot_8to64:
; CHECK-SVE2: // %bb.0: // %entry
; CHECK-SVE2-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-NEXT: ptrue p0.d
; CHECK-SVE2-NEXT: udot z4.s, z2.b, z3.b
-; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-NEXT: uadalp z0.d, p0/m, z4.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-I8MM-LABEL: udot_8to64:
; CHECK-SVE2-I8MM: // %bb.0: // %entry
; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: udot z4.s, z2.b, z3.b
-; CHECK-SVE2-I8MM-NEXT: uaddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-I8MM-NEXT: uaddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-I8MM-NEXT: uadalp z0.d, p0/m, z4.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: udot_8to64:
; CHECK-SME: // %bb.0: // %entry
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: udot z4.s, z2.b, z3.b
-; CHECK-SME-NEXT: uaddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: uaddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: uadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
entry:
%a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>
@@ -210,25 +210,25 @@ define <vscale x 4 x i64> @sdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8
; CHECK-SVE2-LABEL: sdot_8to64:
; CHECK-SVE2: // %bb.0: // %entry
; CHECK-SVE2-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-NEXT: ptrue p0.d
; CHECK-SVE2-NEXT: sdot z4.s, z2.b, z3.b
-; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-I8MM-LABEL: sdot_8to64:
; CHECK-SVE2-I8MM: // %bb.0: // %entry
; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: sdot z4.s, z2.b, z3.b
-; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-I8MM-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: sdot_8to64:
; CHECK-SME: // %bb.0: // %entry
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: sdot z4.s, z2.b, z3.b
-; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
entry:
%a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
@@ -284,17 +284,17 @@ define <vscale x 4 x i64> @usdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i
; CHECK-SVE2-I8MM-LABEL: usdot_8to64:
; CHECK-SVE2-I8MM: // %bb.0: // %entry
; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: usdot z4.s, z2.b, z3.b
-; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-I8MM-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: usdot_8to64:
; CHECK-SME: // %bb.0: // %entry
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: usdot z4.s, z2.b, z3.b
-; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
entry:
%a.wide = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>
@@ -350,17 +350,17 @@ define <vscale x 4 x i64> @sudot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i
; CHECK-SVE2-I8MM-LABEL: sudot_8to64:
; CHECK-SVE2-I8MM: // %bb.0: // %entry
; CHECK-SVE2-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: usdot z4.s, z3.b, z2.b
-; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SVE2-I8MM-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: sudot_8to64:
; CHECK-SME: // %bb.0: // %entry
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: usdot z4.s, z3.b, z2.b
-; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
entry:
%a.wide = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
@@ -470,27 +470,27 @@ define <vscale x 4 x i64> @udot_no_bin_op_8to64(<vscale x 4 x i64> %acc, <vscale
; CHECK-SVE2: // %bb.0:
; CHECK-SVE2-NEXT: movi v3.2d, #0000000000000000
; CHECK-SVE2-NEXT: mov z4.b, #1 // =0x1
+; CHECK-SVE2-NEXT: ptrue p0.d
; CHECK-SVE2-NEXT: udot z3.s, z2.b, z4.b
-; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z3.s
-; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z3.s
+; CHECK-SVE2-NEXT: uadalp z0.d, p0/m, z3.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-I8MM-LABEL: udot_no_bin_op_8to64:
; CHECK-SVE2-I8MM: // %bb.0:
; CHECK-SVE2-I8MM-NEXT: movi v3.2d, #0000000000000000
; CHECK-SVE2-I8MM-NEXT: mov z4.b, #1 // =0x1
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: udot z3.s, z2.b, z4.b
-; CHECK-SVE2-I8MM-NEXT: uaddwb z0.d, z0.d, z3.s
-; CHECK-SVE2-I8MM-NEXT: uaddwt z0.d, z0.d, z3.s
+; CHECK-SVE2-I8MM-NEXT: uadalp z0.d, p0/m, z3.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: udot_no_bin_op_8to64:
; CHECK-SME: // %bb.0:
; CHECK-SME-NEXT: mov z3.b, #1 // =0x1
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: udot z4.s, z2.b, z3.b
-; CHECK-SME-NEXT: uaddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: uaddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: uadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
%a.ext = zext <vscale x 16 x i8> %a to <vscale x 16 x i64>
%partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(<vscale x 4 x i64> %acc, <vscale x 16 x i64> %a.ext)
@@ -502,27 +502,27 @@ define <vscale x 4 x i64> @sdot_no_bin_op_8to64(<vscale x 4 x i64> %acc, <vscale
; CHECK-SVE2: // %bb.0:
; CHECK-SVE2-NEXT: movi v3.2d, #0000000000000000
; CHECK-SVE2-NEXT: mov z4.b, #1 // =0x1
+; CHECK-SVE2-NEXT: ptrue p0.d
; CHECK-SVE2-NEXT: sdot z3.s, z2.b, z4.b
-; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z3.s
-; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z3.s
+; CHECK-SVE2-NEXT: sadalp z0.d, p0/m, z3.s
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-I8MM-LABEL: sdot_no_bin_op_8to64:
; CHECK-SVE2-I8MM: // %bb.0:
; CHECK-SVE2-I8MM-NEXT: movi v3.2d, #0000000000000000
; CHECK-SVE2-I8MM-NEXT: mov z4.b, #1 // =0x1
+; CHECK-SVE2-I8MM-NEXT: ptrue p0.d
; CHECK-SVE2-I8MM-NEXT: sdot z3.s, z2.b, z4.b
-; CHECK-SVE2-I8MM-NEXT: saddwb z0.d, z0.d, z3.s
-; CHECK-SVE2-I8MM-NEXT: saddwt z0.d, z0.d, z3.s
+; CHECK-SVE2-I8MM-NEXT: sadalp z0.d, p0/m, z3.s
; CHECK-SVE2-I8MM-NEXT: ret
;
; CHECK-SME-LABEL: sdot_no_bin_op_8to64:
; CHECK-SME: // %bb.0:
; CHECK-SME-NEXT: mov z3.b, #1 // =0x1
; CHECK-SME-NEXT: mov z4.s, #0 // =0x0
+; CHECK-SME-NEXT: ptrue p0.d
; CHECK-SME-NEXT: sdot z4.s, z2.b, z3.b
-; CHECK-SME-NEXT: saddwb z0.d, z0.d, z4.s
-; CHECK-SME-NEXT: saddwt z0.d, z0.d, z4.s
+; CHECK-SME-NEXT: sadalp z0.d, p0/m, z4.s
; CHECK-SME-NEXT: ret
%a.ext = sext <vscale x 16 x i8> %a to <vscale x 16 x i64>
%partial.reduce = tail call <vscale x 4 x i64> @llvm.vector.partial.reduce.add.nxv4i64.nxv16i64(<vscale x 4 x i64> %acc, <vscale x 16 x i64> %a.ext)
More information about the llvm-commits
mailing list