[llvm] [SelectionDAG][AArch64] Add dot product lowering in NEON for PARTIAL_REDUCE_*MLA ISD nodes (PR #140075)
Nicholas Guy via llvm-commits
llvm-commits at lists.llvm.org
Thu May 22 03:55:02 PDT 2025
https://github.com/NickGuy-Arm updated https://github.com/llvm/llvm-project/pull/140075
>From 1ed6ae3610d5cfb46e22901f6363eb4f98b189f7 Mon Sep 17 00:00:00 2001
From: James Chesterman <james.chesterman at arm.com>
Date: Fri, 28 Feb 2025 17:39:39 +0000
Subject: [PATCH 1/6] [SelectionDAG][AArch64] Add dot product lowering in NEON
for PARTIAL_REDUCE_*MLA ISD nodes
Lowering for fixed width vectors added to tablegen.
There is also custom lowering to ensure that the USDOT patterns are
still lowered for fixed width vectors. It also ensures that the
v16i8 -> v4i64 partial reduction case is lowered here instead of
being split (as there is not a v2i64 dot product instruction).
---
.../Target/AArch64/AArch64ISelLowering.cpp | 16 +++-
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 11 +++
.../neon-partial-reduce-dot-product.ll | 96 ++++++++++++++-----
3 files changed, 98 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 13fb6a32233fe..93ff5d44c3df3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7744,7 +7744,11 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return LowerVECTOR_HISTOGRAM(Op, DAG);
case ISD::PARTIAL_REDUCE_SMLA:
case ISD::PARTIAL_REDUCE_UMLA:
- return LowerPARTIAL_REDUCE_MLA(Op, DAG);
+ case ISD::PARTIAL_REDUCE_SMLA: {
+ if (SDValue Result = LowerPARTIAL_REDUCE_MLA(Op, DAG))
+ return Result;
+ return expandPartialReduceMLA(Op.getNode(), DAG);
+ }
}
}
@@ -27569,6 +27573,15 @@ void AArch64TargetLowering::ReplaceNodeResults(
if (SDValue Res = LowerVECTOR_COMPRESS(SDValue(N, 0), DAG))
Results.push_back(Res);
return;
+ case ISD::PARTIAL_REDUCE_UMLA:
+ case ISD::PARTIAL_REDUCE_SMLA: {
+ SDValue Res;
+ if (Res = LowerPARTIAL_REDUCE_MLA(SDValue(N, 0), DAG))
+ Results.push_back(Res);
+ else
+ Results.push_back(expandPartialReduceMLA(N, DAG));
+ return;
+ }
case ISD::ADD:
case ISD::FADD:
ReplaceAddWithADDP(N, Results, DAG, Subtarget);
@@ -29524,6 +29537,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
+
SDLoc DL(Op);
SDValue Acc = Op.getOperand(0);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index b02a907f7439f..5cc6a38d55977 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1474,6 +1474,17 @@ defm SDOTlane : SIMDThreeSameVectorDotIndex<0, 0, 0b10, "sdot", AArch64sdot>;
defm UDOTlane : SIMDThreeSameVectorDotIndex<1, 0, 0b10, "udot", AArch64udot>;
}
+let Predicates = [HasNEON, HasDotProd] in {
+ def : Pat<(v4i32 (partial_reduce_umla (v4i32 V128:$Acc), (v16i8 V128:$MulLHS), (v16i8 V128:$MulRHS))),
+ (v4i32 (UDOTv16i8 V128:$Acc, V128:$MulLHS, V128:$MulRHS))>;
+ def : Pat<(v4i32 (partial_reduce_smla (v4i32 V128:$Acc), (v16i8 V128:$MulLHS), (v16i8 V128:$MulRHS))),
+ (v4i32 (SDOTv16i8 V128:$Acc, V128:$MulLHS, V128:$MulRHS))>;
+ def : Pat<(v2i32 (partial_reduce_umla (v2i32 V64:$Acc), (v8i8 V64:$MulLHS), (v8i8 V64:$MulRHS))),
+ (v2i32 (UDOTv8i8 V64:$Acc, V64:$MulLHS, V64:$MulRHS))>;
+ def : Pat<(v2i32 (partial_reduce_smla (v2i32 V64:$Acc), (v8i8 V64:$MulLHS), (v8i8 V64:$MulRHS))),
+ (v2i32 (SDOTv8i8 V64:$Acc, V64:$MulLHS, V64:$MulRHS))>;
+} // End HasNEON, HasDotProd
+
// ARMv8.6-A BFloat
let Predicates = [HasNEON, HasBF16] in {
defm BFDOT : SIMDThreeSameVectorBFDot<1, "bfdot">;
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index ab9813aa796e3..03d87d6fea6b3 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -2,7 +2,8 @@
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-NOI8MM
; RUN: llc -mtriple aarch64 -mattr=+neon < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOI8MM,CHECK-NODOT
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM
-; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOI8MM
+; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM,CHECK-NEWLOWERING-I8MM
+; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-NOI8MM,CHECK-NEWLOWERING-NOI8MM
define <4 x i32> @udot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {
; CHECK-DOT-LABEL: udot:
@@ -49,17 +50,20 @@ define <4 x i32> @udot_in_loop(ptr %p1, ptr %p2){
; CHECK-NODOT-NEXT: mov x8, xzr
; CHECK-NODOT-NEXT: .LBB1_1: // %vector.body
; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NODOT-NEXT: ldr q2, [x0, x8]
-; CHECK-NODOT-NEXT: ldr q3, [x1, x8]
-; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT: ldr q0, [x0, x8]
+; CHECK-NODOT-NEXT: ldr q2, [x1, x8]
; CHECK-NODOT-NEXT: add x8, x8, #16
; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b
; CHECK-NODOT-NEXT: cmp x8, #16
-; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v4.4h
-; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v4.8h
-; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v2.4h
-; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v2.8h
+; CHECK-NODOT-NEXT: umull v3.8h, v0.8b, v2.8b
+; CHECK-NODOT-NEXT: umull2 v2.8h, v0.16b, v2.16b
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT: ushll v1.4s, v2.4h, #0
+; CHECK-NODOT-NEXT: uaddw v4.4s, v0.4s, v3.4h
+; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v3.8h
+; CHECK-NODOT-NEXT: uaddw2 v2.4s, v4.4s, v2.8h
+; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
; CHECK-NODOT-NEXT: b.ne .LBB1_1
; CHECK-NODOT-NEXT: // %bb.2: // %end
; CHECK-NODOT-NEXT: ret
@@ -563,22 +567,6 @@ define <4 x i32> @udot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){
}
define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
-; CHECK-DOT-LABEL: udot_no_bin_op_in_loop:
-; CHECK-DOT: // %bb.0: // %entry
-; CHECK-DOT-NEXT: movi v1.2d, #0000000000000000
-; CHECK-DOT-NEXT: movi v2.16b, #1
-; CHECK-DOT-NEXT: mov x8, xzr
-; CHECK-DOT-NEXT: .LBB16_1: // %vector.body
-; CHECK-DOT-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-DOT-NEXT: ldr q3, [x0, x8]
-; CHECK-DOT-NEXT: mov v0.16b, v1.16b
-; CHECK-DOT-NEXT: add x8, x8, #16
-; CHECK-DOT-NEXT: cmp x8, #16
-; CHECK-DOT-NEXT: udot v1.4s, v3.16b, v2.16b
-; CHECK-DOT-NEXT: b.ne .LBB16_1
-; CHECK-DOT-NEXT: // %bb.2: // %end
-; CHECK-DOT-NEXT: ret
-;
; CHECK-NODOT-LABEL: udot_no_bin_op_in_loop:
; CHECK-NODOT: // %bb.0: // %entry
; CHECK-NODOT-NEXT: movi v1.2d, #0000000000000000
@@ -598,6 +586,66 @@ define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
; CHECK-NODOT-NEXT: b.ne .LBB16_1
; CHECK-NODOT-NEXT: // %bb.2: // %end
; CHECK-NODOT-NEXT: ret
+;
+; CHECK-NEWLOWERING-I8MM-LABEL: udot_no_bin_op_in_loop:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: adrp x8, .LCPI16_0
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: adrp x9, .LCPI16_2
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-NEWLOWERING-I8MM-NEXT: adrp x8, .LCPI16_1
+; CHECK-NEWLOWERING-I8MM-NEXT: adrp x10, .LCPI16_3
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x9, :lo12:.LCPI16_2]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q5, [x10, :lo12:.LCPI16_3]
+; CHECK-NEWLOWERING-I8MM-NEXT: mov x8, xzr
+; CHECK-NEWLOWERING-I8MM-NEXT: .LBB16_1: // %vector.body
+; CHECK-NEWLOWERING-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q6, [x0, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: mov v0.16b, v4.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: add x8, x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: cmp x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v7.16b, { v6.16b }, v2.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v4.16b, { v6.16b }, v1.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v16.16b, { v6.16b }, v3.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v6.16b, { v6.16b }, v5.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: add v7.4s, v0.4s, v7.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v6.4s, v6.4s, v16.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v4.4s, v4.4s, v7.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: b.ne .LBB16_1
+; CHECK-NEWLOWERING-I8MM-NEXT: // %bb.2: // %end
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
+;
+; CHECK-NEWLOWERING-NOI8MM-LABEL: udot_no_bin_op_in_loop:
+; CHECK-NEWLOWERING-NOI8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x8, .LCPI16_0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x9, .LCPI16_2
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x8, .LCPI16_1
+; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x10, .LCPI16_3
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q3, [x9, :lo12:.LCPI16_2]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q5, [x10, :lo12:.LCPI16_3]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: mov x8, xzr
+; CHECK-NEWLOWERING-NOI8MM-NEXT: .LBB16_1: // %vector.body
+; CHECK-NEWLOWERING-NOI8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q6, [x0, x8]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: mov v0.16b, v4.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add x8, x8, #16
+; CHECK-NEWLOWERING-NOI8MM-NEXT: cmp x8, #16
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v7.16b, { v6.16b }, v2.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v4.16b, { v6.16b }, v1.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v16.16b, { v6.16b }, v3.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v6.16b, { v6.16b }, v5.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v7.4s, v0.4s, v7.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v6.4s, v6.4s, v16.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v4.4s, v4.4s, v7.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: b.ne .LBB16_1
+; CHECK-NEWLOWERING-NOI8MM-NEXT: // %bb.2: // %end
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
entry:
br label %vector.body
>From 3a1ce170c3110117083d83c1708e1c097fba2299 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Tue, 13 May 2025 18:05:59 +0100
Subject: [PATCH 2/6] Fix merge issues
---
.../Target/AArch64/AArch64ISelLowering.cpp | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 93ff5d44c3df3..00deccfa60144 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29537,6 +29537,11 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
+ bool Scalable = Op.getValueType().isScalableVector();
+ if (Scalable && !Subtarget->isSVEorStreamingSVEAvailable())
+ return SDValue();
+ if (!Scalable && (!Subtarget->isNeonAvailable() || !Subtarget->hasDotProd()))
+ return SDValue();
SDLoc DL(Op);
@@ -29544,13 +29549,19 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SDValue LHS = Op.getOperand(1);
SDValue RHS = Op.getOperand(2);
EVT ResultVT = Op.getValueType();
- assert(ResultVT == MVT::nxv2i64 && LHS.getValueType() == MVT::nxv16i8);
- SDValue DotNode = DAG.getNode(Op.getOpcode(), DL, MVT::nxv4i32,
- DAG.getConstant(0, DL, MVT::nxv4i32), LHS, RHS);
+ assert((Scalable && ResultVT == MVT::nxv2i64 &&
+ LHS.getValueType() == MVT::nxv16i8) ||
+ (!Scalable && ResultVT == MVT::v2i64 &&
+ LHS.getValueType() == MVT::v16i8));
+
+ EVT DotVT = Scalable ? MVT::nxv4i32 : MVT::v4i32;
+ SDValue DotNode = DAG.getNode(Op.getOpcode(), DL, DotVT,
+ DAG.getConstant(0, DL, DotVT), LHS, RHS);
bool IsUnsigned = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA;
- if (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable()) {
+ if (Scalable &&
+ (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
unsigned LoOpcode = IsUnsigned ? AArch64ISD::UADDWB : AArch64ISD::SADDWB;
unsigned HiOpcode = IsUnsigned ? AArch64ISD::UADDWT : AArch64ISD::SADDWT;
SDValue Lo = DAG.getNode(LoOpcode, DL, ResultVT, Acc, DotNode);
>From 6e40a1e00d8e8b25adbdfea247aafa3b34789873 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Thu, 15 May 2025 15:57:02 +0100
Subject: [PATCH 3/6] Fix up neon 8to64 cases
---
.../Target/AArch64/AArch64ISelLowering.cpp | 46 +-
.../neon-partial-reduce-dot-product.ll | 457 ++++++++++++------
2 files changed, 345 insertions(+), 158 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 00deccfa60144..f1354bf1147dd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1872,6 +1872,15 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MVT::nxv2i64, MVT::nxv16i8, Custom);
}
+ if (EnablePartialReduceNodes && Subtarget->hasNEON() &&
+ Subtarget->hasDotProd()) {
+ setPartialReduceMLAAction(MVT::v2i64, MVT::v8i16, Legal);
+ setPartialReduceMLAAction(MVT::v4i32, MVT::v16i8, Legal);
+ setPartialReduceMLAAction(MVT::v4i32, MVT::v16i8, Legal);
+ setPartialReduceMLAAction(MVT::v2i32, MVT::v8i8, Legal);
+ setPartialReduceMLAAction(MVT::v2i64, MVT::v16i8, Custom);
+ }
+
// Handle operations that are only available in non-streaming SVE mode.
if (Subtarget->isSVEAvailable()) {
for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64,
@@ -7743,8 +7752,7 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
return LowerVECTOR_HISTOGRAM(Op, DAG);
case ISD::PARTIAL_REDUCE_SMLA:
- case ISD::PARTIAL_REDUCE_UMLA:
- case ISD::PARTIAL_REDUCE_SMLA: {
+ case ISD::PARTIAL_REDUCE_UMLA: {
if (SDValue Result = LowerPARTIAL_REDUCE_MLA(Op, DAG))
return Result;
return expandPartialReduceMLA(Op.getNode(), DAG);
@@ -27575,8 +27583,7 @@ void AArch64TargetLowering::ReplaceNodeResults(
return;
case ISD::PARTIAL_REDUCE_UMLA:
case ISD::PARTIAL_REDUCE_SMLA: {
- SDValue Res;
- if (Res = LowerPARTIAL_REDUCE_MLA(SDValue(N, 0), DAG))
+ if (SDValue Res = LowerPARTIAL_REDUCE_MLA(SDValue(N, 0), DAG))
Results.push_back(Res);
else
Results.push_back(expandPartialReduceMLA(N, DAG));
@@ -29531,9 +29538,9 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
}
/// If a PARTIAL_REDUCE_MLA node comes in with an accumulator-input type pairing
-/// of nxv2i64/nxv16i8, we cannot directly lower it to a (u|s)dot. We can
+/// of v2i64/v16i8, we cannot directly lower it to a (u|s)dot. We can
/// however still make use of the dot product instruction by instead
-/// accumulating over two steps: nxv16i8 -> nxv4i32 -> nxv2i64.
+/// accumulating over two steps: v16i8 -> v4i32 -> v2i64.
SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
@@ -29568,12 +29575,27 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
}
- unsigned LoOpcode = IsUnsigned ? AArch64ISD::UUNPKLO : AArch64ISD::SUNPKLO;
- unsigned HiOpcode = IsUnsigned ? AArch64ISD::UUNPKHI : AArch64ISD::SUNPKHI;
- auto Lo = DAG.getNode(LoOpcode, DL, ResultVT, DotNode);
- auto Hi = DAG.getNode(HiOpcode, DL, ResultVT, DotNode);
- auto Extended = DAG.getNode(ISD::ADD, DL, ResultVT, Lo, Hi);
- return DAG.getNode(ISD::ADD, DL, ResultVT, Acc, Extended);
+ if (Scalable) {
+ unsigned LoOpcode = IsUnsigned ? AArch64ISD::UUNPKLO : AArch64ISD::SUNPKLO;
+ unsigned HiOpcode = IsUnsigned ? AArch64ISD::UUNPKHI : AArch64ISD::SUNPKHI;
+ auto Lo = DAG.getNode(LoOpcode, DL, ResultVT, DotNode);
+ auto Hi = DAG.getNode(HiOpcode, DL, ResultVT, DotNode);
+ auto Extended = DAG.getNode(ISD::ADD, DL, ResultVT, Lo, Hi);
+ return DAG.getNode(ISD::ADD, DL, ResultVT, Acc, Extended);
+ }
+
+ // Fold v4i32 into v2i64
+ // SDValues
+ auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(DotNode, DL);
+ if (IsUnsigned) {
+ DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, MVT::v2i64);
+ DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, MVT::v2i64);
+ } else {
+ DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, MVT::v2i64);
+ DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, MVT::v2i64);
+ }
+ auto Lo = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Acc, DotNodeLo);
+ return DAG.getNode(ISD::ADD, DL, MVT::v2i64, Lo, DotNodeHi);
}
SDValue
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 03d87d6fea6b3..47a4796d0f9a1 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -50,20 +50,17 @@ define <4 x i32> @udot_in_loop(ptr %p1, ptr %p2){
; CHECK-NODOT-NEXT: mov x8, xzr
; CHECK-NODOT-NEXT: .LBB1_1: // %vector.body
; CHECK-NODOT-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NODOT-NEXT: ldr q0, [x0, x8]
-; CHECK-NODOT-NEXT: ldr q2, [x1, x8]
+; CHECK-NODOT-NEXT: ldr q2, [x0, x8]
+; CHECK-NODOT-NEXT: ldr q3, [x1, x8]
+; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
; CHECK-NODOT-NEXT: add x8, x8, #16
; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b
; CHECK-NODOT-NEXT: umull2 v2.8h, v2.16b, v3.16b
; CHECK-NODOT-NEXT: cmp x8, #16
-; CHECK-NODOT-NEXT: umull v3.8h, v0.8b, v2.8b
-; CHECK-NODOT-NEXT: umull2 v2.8h, v0.16b, v2.16b
-; CHECK-NODOT-NEXT: mov v0.16b, v1.16b
-; CHECK-NODOT-NEXT: ushll v1.4s, v2.4h, #0
-; CHECK-NODOT-NEXT: uaddw v4.4s, v0.4s, v3.4h
-; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v3.8h
-; CHECK-NODOT-NEXT: uaddw2 v2.4s, v4.4s, v2.8h
-; CHECK-NODOT-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v4.4h
+; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v4.8h
+; CHECK-NODOT-NEXT: uaddw v1.4s, v1.4s, v2.4h
+; CHECK-NODOT-NEXT: uaddw2 v1.4s, v1.4s, v2.8h
; CHECK-NODOT-NEXT: b.ne .LBB1_1
; CHECK-NODOT-NEXT: // %bb.2: // %end
; CHECK-NODOT-NEXT: ret
@@ -178,10 +175,17 @@ define <4 x i32> @usdot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {
; CHECK-NOI8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: usdot:
-; CHECK-I8MM: // %bb.0:
-; CHECK-I8MM-NEXT: usdot v0.4s, v1.16b, v2.16b
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: usdot:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v1.8h, v1.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v4.4h, v3.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v4.8h, v3.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
%u.wide = zext <16 x i8> %u to <16 x i32>
%s.wide = sext <16 x i8> %s to <16 x i32>
%mult = mul nuw nsw <16 x i32> %s.wide, %u.wide
@@ -213,21 +217,28 @@ define <4 x i32> @usdot_in_loop(ptr %p1, ptr %p2){
; CHECK-NOI8MM-NEXT: // %bb.2: // %end
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: usdot_in_loop:
-; CHECK-I8MM: // %bb.0: // %entry
-; CHECK-I8MM-NEXT: movi v1.2d, #0000000000000000
-; CHECK-I8MM-NEXT: mov x8, xzr
-; CHECK-I8MM-NEXT: .LBB6_1: // %vector.body
-; CHECK-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-I8MM-NEXT: ldr q2, [x0, x8]
-; CHECK-I8MM-NEXT: ldr q3, [x1, x8]
-; CHECK-I8MM-NEXT: mov v0.16b, v1.16b
-; CHECK-I8MM-NEXT: add x8, x8, #16
-; CHECK-I8MM-NEXT: usdot v1.4s, v3.16b, v2.16b
-; CHECK-I8MM-NEXT: cmp x8, #16
-; CHECK-I8MM-NEXT: b.ne .LBB6_1
-; CHECK-I8MM-NEXT: // %bb.2: // %end
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: usdot_in_loop:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: mov x8, xzr
+; CHECK-NEWLOWERING-I8MM-NEXT: .LBB6_1: // %vector.body
+; CHECK-NEWLOWERING-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: add x8, x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v5.8h, v3.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: cmp x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v4.4h, v5.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v4.8h, v5.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v2.4h, v3.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v2.8h, v3.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: b.ne .LBB6_1
+; CHECK-NEWLOWERING-I8MM-NEXT: // %bb.2: // %end
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
entry:
br label %vector.body
@@ -268,10 +279,22 @@ define <2 x i32> @usdot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: usdot_narrow:
-; CHECK-I8MM: // %bb.0:
-; CHECK-I8MM-NEXT: usdot v0.2s, v1.8b, v2.8b
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: usdot_narrow:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v2.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEWLOWERING-I8MM-NEXT: smull v3.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: smull2 v1.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: add v0.2s, v3.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v5.4h, v4.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: add v0.2s, v1.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
%u.wide = zext <8 x i8> %u to <8 x i32>
%s.wide = sext <8 x i8> %s to <8 x i32>
%mult = mul nuw nsw <8 x i32> %s.wide, %u.wide
@@ -292,10 +315,17 @@ define <4 x i32> @sudot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: sudot:
-; CHECK-I8MM: // %bb.0:
-; CHECK-I8MM-NEXT: usdot v0.4s, v2.16b, v1.16b
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: sudot:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v1.8h, v1.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v4.4h, v3.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v4.8h, v3.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
%s.wide = sext <16 x i8> %u to <16 x i32>
%u.wide = zext <16 x i8> %s to <16 x i32>
%mult = mul nuw nsw <16 x i32> %u.wide, %s.wide
@@ -327,21 +357,28 @@ define <4 x i32> @sudot_in_loop(ptr %p1, ptr %p2){
; CHECK-NOI8MM-NEXT: // %bb.2: // %end
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: sudot_in_loop:
-; CHECK-I8MM: // %bb.0: // %entry
-; CHECK-I8MM-NEXT: movi v1.2d, #0000000000000000
-; CHECK-I8MM-NEXT: mov x8, xzr
-; CHECK-I8MM-NEXT: .LBB9_1: // %vector.body
-; CHECK-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-I8MM-NEXT: ldr q2, [x0, x8]
-; CHECK-I8MM-NEXT: ldr q3, [x1, x8]
-; CHECK-I8MM-NEXT: mov v0.16b, v1.16b
-; CHECK-I8MM-NEXT: add x8, x8, #16
-; CHECK-I8MM-NEXT: usdot v1.4s, v2.16b, v3.16b
-; CHECK-I8MM-NEXT: cmp x8, #16
-; CHECK-I8MM-NEXT: b.ne .LBB9_1
-; CHECK-I8MM-NEXT: // %bb.2: // %end
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: sudot_in_loop:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: mov x8, xzr
+; CHECK-NEWLOWERING-I8MM-NEXT: .LBB9_1: // %vector.body
+; CHECK-NEWLOWERING-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: add x8, x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v5.8h, v3.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: cmp x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v4.4h, v5.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v4.8h, v5.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v2.4h, v3.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v2.8h, v3.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: b.ne .LBB9_1
+; CHECK-NEWLOWERING-I8MM-NEXT: // %bb.2: // %end
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
entry:
br label %vector.body
@@ -382,10 +419,22 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: sudot_narrow:
-; CHECK-I8MM: // %bb.0:
-; CHECK-I8MM-NEXT: usdot v0.2s, v2.8b, v1.8b
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: sudot_narrow:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v2.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEWLOWERING-I8MM-NEXT: smull v3.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v2.4h, v1.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v4.16b, v1.16b, v1.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v5.16b, v2.16b, v2.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: smull2 v1.4s, v2.8h, v1.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v3.16b, v3.16b, v3.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: ext v1.16b, v1.16b, v1.16b, #8
+; CHECK-NEWLOWERING-I8MM-NEXT: add v0.2s, v3.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v5.4h, v4.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: add v0.2s, v1.2s, v0.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
%u.wide = sext <8 x i8> %u to <8 x i32>
%s.wide = zext <8 x i8> %s to <8 x i32>
%mult = mul nuw nsw <8 x i32> %s.wide, %u.wide
@@ -394,14 +443,6 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
}
define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
-; CHECK-DOT-LABEL: udot_8to64:
-; CHECK-DOT: // %bb.0: // %entry
-; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: ret
-;
; CHECK-NODOT-LABEL: udot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
; CHECK-NODOT-NEXT: umull v4.8h, v2.8b, v3.8b
@@ -419,6 +460,22 @@ define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
+;
+; CHECK-NEWLOWERING-I8MM-LABEL: udot_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
+;
+; CHECK-NEWLOWERING-NOI8MM-LABEL: udot_8to64:
+; CHECK-NEWLOWERING-NOI8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-NOI8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-NOI8MM-NEXT: udot v4.4s, v2.16b, v3.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
entry:
%a.wide = zext <16 x i8> %a to <16 x i64>
%b.wide = zext <16 x i8> %b to <16 x i64>
@@ -429,14 +486,6 @@ entry:
}
define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
-; CHECK-DOT-LABEL: sdot_8to64:
-; CHECK-DOT: // %bb.0: // %entry
-; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: ret
-;
; CHECK-NODOT-LABEL: sdot_8to64:
; CHECK-NODOT: // %bb.0: // %entry
; CHECK-NODOT-NEXT: smull v4.8h, v2.8b, v3.8b
@@ -454,6 +503,22 @@ define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s
; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
+;
+; CHECK-NEWLOWERING-I8MM-LABEL: sdot_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
+;
+; CHECK-NEWLOWERING-NOI8MM-LABEL: sdot_8to64:
+; CHECK-NEWLOWERING-NOI8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-NOI8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
entry:
%a.wide = sext <16 x i8> %a to <16 x i64>
%b.wide = sext <16 x i8> %b to <16 x i64>
@@ -488,13 +553,29 @@ define <4 x i64> @usdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NOI8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: usdot_8to64:
-; CHECK-I8MM: // %bb.0: // %entry
-; CHECK-I8MM-NEXT: movi v4.2d, #0000000000000000
-; CHECK-I8MM-NEXT: usdot v4.4s, v2.16b, v3.16b
-; CHECK-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: usdot_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v5.8h, v3.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v6.4s, v4.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v7.4s, v2.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v16.4s, v5.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v17.4s, v3.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v4.4s, v4.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v5.4s, v5.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.2d, v6.2s, v16.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.2d, v7.2s, v17.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.2d, v6.4s, v16.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.2d, v7.4s, v17.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.2d, v4.2s, v5.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.2d, v2.2s, v3.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.2d, v4.4s, v5.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
entry:
%a.wide = zext <16 x i8> %a to <16 x i64>
%b.wide = sext <16 x i8> %b to <16 x i64>
@@ -529,13 +610,29 @@ define <4 x i64> @sudot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NOI8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: sudot_8to64:
-; CHECK-I8MM: // %bb.0: // %entry
-; CHECK-I8MM-NEXT: movi v4.2d, #0000000000000000
-; CHECK-I8MM-NEXT: usdot v4.4s, v3.16b, v2.16b
-; CHECK-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: sudot_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v4.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v5.8h, v3.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v6.4s, v4.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v7.4s, v2.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v16.4s, v5.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v17.4s, v3.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v4.4s, v4.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.2d, v6.2s, v16.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.2d, v7.2s, v17.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.2d, v6.4s, v16.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.2d, v7.4s, v17.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.2d, v4.2s, v5.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.2d, v2.2s, v3.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.2d, v4.4s, v5.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
entry:
%a.wide = sext <16 x i8> %a to <16 x i64>
%b.wide = zext <16 x i8> %b to <16 x i64>
@@ -590,29 +687,29 @@ define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
; CHECK-NEWLOWERING-I8MM-LABEL: udot_no_bin_op_in_loop:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: adrp x8, .LCPI16_0
-; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEWLOWERING-I8MM-NEXT: adrp x9, .LCPI16_2
; CHECK-NEWLOWERING-I8MM-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
; CHECK-NEWLOWERING-I8MM-NEXT: adrp x8, .LCPI16_1
; CHECK-NEWLOWERING-I8MM-NEXT: adrp x10, .LCPI16_3
-; CHECK-NEWLOWERING-I8MM-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
-; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x9, :lo12:.LCPI16_2]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x8, :lo12:.LCPI16_1]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q4, [x9, :lo12:.LCPI16_2]
; CHECK-NEWLOWERING-I8MM-NEXT: ldr q5, [x10, :lo12:.LCPI16_3]
; CHECK-NEWLOWERING-I8MM-NEXT: mov x8, xzr
; CHECK-NEWLOWERING-I8MM-NEXT: .LBB16_1: // %vector.body
; CHECK-NEWLOWERING-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEWLOWERING-I8MM-NEXT: ldr q6, [x0, x8]
-; CHECK-NEWLOWERING-I8MM-NEXT: mov v0.16b, v4.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: mov v0.16b, v2.16b
; CHECK-NEWLOWERING-I8MM-NEXT: add x8, x8, #16
; CHECK-NEWLOWERING-I8MM-NEXT: cmp x8, #16
-; CHECK-NEWLOWERING-I8MM-NEXT: tbl v7.16b, { v6.16b }, v2.16b
-; CHECK-NEWLOWERING-I8MM-NEXT: tbl v4.16b, { v6.16b }, v1.16b
-; CHECK-NEWLOWERING-I8MM-NEXT: tbl v16.16b, { v6.16b }, v3.16b
-; CHECK-NEWLOWERING-I8MM-NEXT: tbl v6.16b, { v6.16b }, v5.16b
-; CHECK-NEWLOWERING-I8MM-NEXT: add v7.4s, v0.4s, v7.4s
-; CHECK-NEWLOWERING-I8MM-NEXT: add v6.4s, v6.4s, v16.4s
-; CHECK-NEWLOWERING-I8MM-NEXT: add v4.4s, v4.4s, v7.4s
-; CHECK-NEWLOWERING-I8MM-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v7.16b, { v6.16b }, v3.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v16.16b, { v6.16b }, v4.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v17.16b, { v6.16b }, v5.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: tbl v6.16b, { v6.16b }, v1.16b
+; CHECK-NEWLOWERING-I8MM-NEXT: add v2.4s, v2.4s, v17.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v7.4s, v16.4s, v7.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v2.4s, v2.4s, v7.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: add v2.4s, v2.4s, v6.4s
; CHECK-NEWLOWERING-I8MM-NEXT: b.ne .LBB16_1
; CHECK-NEWLOWERING-I8MM-NEXT: // %bb.2: // %end
; CHECK-NEWLOWERING-I8MM-NEXT: ret
@@ -620,29 +717,29 @@ define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
; CHECK-NEWLOWERING-NOI8MM-LABEL: udot_no_bin_op_in_loop:
; CHECK-NEWLOWERING-NOI8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x8, .LCPI16_0
-; CHECK-NEWLOWERING-NOI8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEWLOWERING-NOI8MM-NEXT: movi v2.2d, #0000000000000000
; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x9, .LCPI16_2
; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q1, [x8, :lo12:.LCPI16_0]
; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x8, .LCPI16_1
; CHECK-NEWLOWERING-NOI8MM-NEXT: adrp x10, .LCPI16_3
-; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
-; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q3, [x9, :lo12:.LCPI16_2]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q3, [x8, :lo12:.LCPI16_1]
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q4, [x9, :lo12:.LCPI16_2]
; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q5, [x10, :lo12:.LCPI16_3]
; CHECK-NEWLOWERING-NOI8MM-NEXT: mov x8, xzr
; CHECK-NEWLOWERING-NOI8MM-NEXT: .LBB16_1: // %vector.body
; CHECK-NEWLOWERING-NOI8MM-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEWLOWERING-NOI8MM-NEXT: ldr q6, [x0, x8]
-; CHECK-NEWLOWERING-NOI8MM-NEXT: mov v0.16b, v4.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: mov v0.16b, v2.16b
; CHECK-NEWLOWERING-NOI8MM-NEXT: add x8, x8, #16
; CHECK-NEWLOWERING-NOI8MM-NEXT: cmp x8, #16
-; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v7.16b, { v6.16b }, v2.16b
-; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v4.16b, { v6.16b }, v1.16b
-; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v16.16b, { v6.16b }, v3.16b
-; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v6.16b, { v6.16b }, v5.16b
-; CHECK-NEWLOWERING-NOI8MM-NEXT: add v7.4s, v0.4s, v7.4s
-; CHECK-NEWLOWERING-NOI8MM-NEXT: add v6.4s, v6.4s, v16.4s
-; CHECK-NEWLOWERING-NOI8MM-NEXT: add v4.4s, v4.4s, v7.4s
-; CHECK-NEWLOWERING-NOI8MM-NEXT: add v4.4s, v6.4s, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v7.16b, { v6.16b }, v3.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v16.16b, { v6.16b }, v4.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v17.16b, { v6.16b }, v5.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: tbl v6.16b, { v6.16b }, v1.16b
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v2.4s, v2.4s, v17.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v7.4s, v16.4s, v7.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v2.4s, v2.4s, v7.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: add v2.4s, v2.4s, v6.4s
; CHECK-NEWLOWERING-NOI8MM-NEXT: b.ne .LBB16_1
; CHECK-NEWLOWERING-NOI8MM-NEXT: // %bb.2: // %end
; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
@@ -739,15 +836,6 @@ define <2 x i32> @sdot_no_bin_op_narrow(<2 x i32> %acc, <8 x i8> %a){
}
define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
-; CHECK-DOT-LABEL: udot_no_bin_op_8to64:
-; CHECK-DOT: // %bb.0:
-; CHECK-DOT-NEXT: movi v3.16b, #1
-; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-DOT-NEXT: udot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: ret
-;
; CHECK-NODOT-LABEL: udot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: ushll v3.8h, v2.8b, #0
@@ -765,21 +853,48 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
+;
+; CHECK-NEWLOWERING-I8MM-LABEL: udot_no_bin_op_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v3.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v5.4s, v2.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw v1.2d, v1.2d, v5.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw2 v1.2d, v1.2d, v5.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw v1.2d, v1.2d, v2.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw v0.2d, v0.2d, v3.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
+;
+; CHECK-NEWLOWERING-NOI8MM-LABEL: udot_no_bin_op_8to64:
+; CHECK-NEWLOWERING-NOI8MM: // %bb.0:
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll v3.8h, v2.8b, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll v4.4s, v3.4h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll v5.4s, v2.4h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw v1.2d, v1.2d, v5.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw2 v1.2d, v1.2d, v5.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw v1.2d, v1.2d, v2.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw v0.2d, v0.2d, v3.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw2 v1.2d, v1.2d, v2.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
%a.wide = zext <16 x i8> %a to <16 x i64>
%partial.reduce = tail call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)
ret <4 x i64> %partial.reduce
}
define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
-; CHECK-DOT-LABEL: sdot_no_bin_op_8to64:
-; CHECK-DOT: // %bb.0:
-; CHECK-DOT-NEXT: movi v3.16b, #1
-; CHECK-DOT-NEXT: movi v4.2d, #0000000000000000
-; CHECK-DOT-NEXT: sdot v4.4s, v2.16b, v3.16b
-; CHECK-DOT-NEXT: saddw2 v1.2d, v1.2d, v4.4s
-; CHECK-DOT-NEXT: saddw v0.2d, v0.2d, v4.2s
-; CHECK-DOT-NEXT: ret
-;
; CHECK-NODOT-LABEL: sdot_no_bin_op_8to64:
; CHECK-NODOT: // %bb.0:
; CHECK-NODOT-NEXT: sshll v3.8h, v2.8b, #0
@@ -797,6 +912,42 @@ define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-NEXT: saddw2 v1.2d, v1.2d, v2.4s
; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
+;
+; CHECK-NEWLOWERING-I8MM-LABEL: sdot_no_bin_op_8to64:
+; CHECK-NEWLOWERING-I8MM: // %bb.0:
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v3.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v4.4s, v3.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v5.4s, v2.4h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw v1.2d, v1.2d, v5.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw2 v1.2d, v1.2d, v5.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw v1.2d, v1.2d, v2.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw v0.2d, v0.2d, v3.2s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw2 v1.2d, v1.2d, v2.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
+;
+; CHECK-NEWLOWERING-NOI8MM-LABEL: sdot_no_bin_op_8to64:
+; CHECK-NEWLOWERING-NOI8MM: // %bb.0:
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll v3.8h, v2.8b, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll v4.4s, v3.4h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll v5.4s, v2.4h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: sshll2 v2.4s, v2.8h, #0
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw v1.2d, v1.2d, v5.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw2 v1.2d, v1.2d, v5.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw2 v0.2d, v0.2d, v4.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw v1.2d, v1.2d, v2.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw v0.2d, v0.2d, v3.2s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw2 v1.2d, v1.2d, v2.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: saddw2 v0.2d, v0.2d, v3.4s
+; CHECK-NEWLOWERING-NOI8MM-NEXT: ret
%a.wide = sext <16 x i8> %a to <16 x i64>
%partial.reduce = tail call <4 x i64> @llvm.experimental.vector.partial.reduce.add.v4i64.v16i64(<4 x i64> %acc, <16 x i64> %a.wide)
ret <4 x i64> %partial.reduce
@@ -950,24 +1101,36 @@ define <4 x i32> @usdot_multiple_zext_users(ptr %p1, ptr %p2, ptr %p3) {
; CHECK-NOI8MM-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-NOI8MM-NEXT: ret
;
-; CHECK-I8MM-LABEL: usdot_multiple_zext_users:
-; CHECK-I8MM: // %bb.0: // %entry
-; CHECK-I8MM-NEXT: movi v0.2d, #0000000000000000
-; CHECK-I8MM-NEXT: movi v1.2d, #0000000000000000
-; CHECK-I8MM-NEXT: mov x8, xzr
-; CHECK-I8MM-NEXT: .LBB28_1: // %vector.body
-; CHECK-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-I8MM-NEXT: ldr q2, [x0, x8]
-; CHECK-I8MM-NEXT: ldr q3, [x1, x8]
-; CHECK-I8MM-NEXT: ldr q4, [x2, x8]
-; CHECK-I8MM-NEXT: add x8, x8, #16
-; CHECK-I8MM-NEXT: usdot v0.4s, v4.16b, v2.16b
-; CHECK-I8MM-NEXT: usdot v1.4s, v4.16b, v3.16b
-; CHECK-I8MM-NEXT: cmp x8, #1024
-; CHECK-I8MM-NEXT: b.ne .LBB28_1
-; CHECK-I8MM-NEXT: // %bb.2: // %end
-; CHECK-I8MM-NEXT: add v0.4s, v1.4s, v0.4s
-; CHECK-I8MM-NEXT: ret
+; CHECK-NEWLOWERING-I8MM-LABEL: usdot_multiple_zext_users:
+; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEWLOWERING-I8MM-NEXT: mov x8, xzr
+; CHECK-NEWLOWERING-I8MM-NEXT: .LBB28_1: // %vector.body
+; CHECK-NEWLOWERING-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: ldr q4, [x2, x8]
+; CHECK-NEWLOWERING-I8MM-NEXT: add x8, x8, #16
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v5.8h, v2.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll v6.8h, v4.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll v7.8h, v3.8b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v2.8h, v2.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: ushll2 v4.8h, v4.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: sshll2 v3.8h, v3.16b, #0
+; CHECK-NEWLOWERING-I8MM-NEXT: cmp x8, #1024
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v5.4h, v6.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v7.4h, v6.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v5.8h, v6.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v7.8h, v6.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v0.4s, v2.4h, v4.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal v1.4s, v3.4h, v4.4h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v0.4s, v2.8h, v4.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: smlal2 v1.4s, v3.8h, v4.8h
+; CHECK-NEWLOWERING-I8MM-NEXT: b.ne .LBB28_1
+; CHECK-NEWLOWERING-I8MM-NEXT: // %bb.2: // %end
+; CHECK-NEWLOWERING-I8MM-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-NEWLOWERING-I8MM-NEXT: ret
entry:
br label %vector.body
@@ -996,3 +1159,5 @@ end:
%2 = add <4 x i32> %psum2, %psum1
ret <4 x i32> %2
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-I8MM: {{.*}}
>From 5ff49a7447f7e6748e704f0c3e580ef6ae477e9a Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Tue, 20 May 2025 16:04:11 +0100
Subject: [PATCH 4/6] Address comments and fix test check
---
.../Target/AArch64/AArch64ISelLowering.cpp | 41 ++----
.../neon-partial-reduce-dot-product.ll | 139 +++++++++++++++++-
2 files changed, 153 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f1354bf1147dd..9660b8331c3b8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7752,11 +7752,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
return LowerVECTOR_HISTOGRAM(Op, DAG);
case ISD::PARTIAL_REDUCE_SMLA:
- case ISD::PARTIAL_REDUCE_UMLA: {
- if (SDValue Result = LowerPARTIAL_REDUCE_MLA(Op, DAG))
- return Result;
- return expandPartialReduceMLA(Op.getNode(), DAG);
- }
+ case ISD::PARTIAL_REDUCE_UMLA:
+ return LowerPARTIAL_REDUCE_MLA(Op, DAG);
}
}
@@ -27585,8 +27582,6 @@ void AArch64TargetLowering::ReplaceNodeResults(
case ISD::PARTIAL_REDUCE_SMLA: {
if (SDValue Res = LowerPARTIAL_REDUCE_MLA(SDValue(N, 0), DAG))
Results.push_back(Res);
- else
- Results.push_back(expandPartialReduceMLA(N, DAG));
return;
}
case ISD::ADD:
@@ -29538,9 +29533,13 @@ SDValue AArch64TargetLowering::LowerVECTOR_HISTOGRAM(SDValue Op,
}
/// If a PARTIAL_REDUCE_MLA node comes in with an accumulator-input type pairing
-/// of v2i64/v16i8, we cannot directly lower it to a (u|s)dot. We can
+/// of (nx)v2i64/(nx)v16i8, we cannot directly lower it to a (u|s)dot. We can
/// however still make use of the dot product instruction by instead
-/// accumulating over two steps: v16i8 -> v4i32 -> v2i64.
+/// accumulating over two steps: (nx)v16i8 -> (nx)v4i32 -> (nx)v2i64.
+/// If available, make use of the (U|S)ADDW(B|T) instructions, otherwise
+/// the following pattern is emitted:
+/// add(add(Acc, ext(EXTRACT_SUBVECTOR(N, 0)), ext(EXTRACT_SUBVECTOR(N,
+/// NTy/2))))
SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
@@ -29575,27 +29574,17 @@ AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
return DAG.getNode(HiOpcode, DL, ResultVT, Lo, DotNode);
}
- if (Scalable) {
- unsigned LoOpcode = IsUnsigned ? AArch64ISD::UUNPKLO : AArch64ISD::SUNPKLO;
- unsigned HiOpcode = IsUnsigned ? AArch64ISD::UUNPKHI : AArch64ISD::SUNPKHI;
- auto Lo = DAG.getNode(LoOpcode, DL, ResultVT, DotNode);
- auto Hi = DAG.getNode(HiOpcode, DL, ResultVT, DotNode);
- auto Extended = DAG.getNode(ISD::ADD, DL, ResultVT, Lo, Hi);
- return DAG.getNode(ISD::ADD, DL, ResultVT, Acc, Extended);
- }
-
- // Fold v4i32 into v2i64
- // SDValues
+ // Fold (nx)v4i32 into (nx)v2i64
auto [DotNodeLo, DotNodeHi] = DAG.SplitVector(DotNode, DL);
if (IsUnsigned) {
- DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, MVT::v2i64);
- DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, MVT::v2i64);
+ DotNodeLo = DAG.getZExtOrTrunc(DotNodeLo, DL, ResultVT);
+ DotNodeHi = DAG.getZExtOrTrunc(DotNodeHi, DL, ResultVT);
} else {
- DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, MVT::v2i64);
- DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, MVT::v2i64);
+ DotNodeLo = DAG.getSExtOrTrunc(DotNodeLo, DL, ResultVT);
+ DotNodeHi = DAG.getSExtOrTrunc(DotNodeHi, DL, ResultVT);
}
- auto Lo = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Acc, DotNodeLo);
- return DAG.getNode(ISD::ADD, DL, MVT::v2i64, Lo, DotNodeHi);
+ auto Lo = DAG.getNode(ISD::ADD, DL, ResultVT, Acc, DotNodeLo);
+ return DAG.getNode(ISD::ADD, DL, ResultVT, Lo, DotNodeHi);
}
SDValue
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 47a4796d0f9a1..1f8c7f0fc74f6 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-NOI8MM
; RUN: llc -mtriple aarch64 -mattr=+neon < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOI8MM,CHECK-NODOT
-; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM
+; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM,CHECK-DOT-I8MM
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM,CHECK-NEWLOWERING-I8MM
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-NOI8MM,CHECK-NEWLOWERING-NOI8MM
@@ -175,6 +175,11 @@ define <4 x i32> @usdot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {
; CHECK-NOI8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: usdot:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v1.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: usdot:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: ushll v3.8h, v1.8b, #0
@@ -217,6 +222,22 @@ define <4 x i32> @usdot_in_loop(ptr %p1, ptr %p2){
; CHECK-NOI8MM-NEXT: // %bb.2: // %end
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: usdot_in_loop:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: mov x8, xzr
+; CHECK-DOT-I8MM-NEXT: .LBB6_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT: add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v3.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: cmp x8, #16
+; CHECK-DOT-I8MM-NEXT: b.ne .LBB6_1
+; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: usdot_in_loop:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: movi v1.2d, #0000000000000000
@@ -279,6 +300,11 @@ define <2 x i32> @usdot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: usdot_narrow:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: usdot v0.2s, v1.8b, v2.8b
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: usdot_narrow:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: ushll v1.8h, v1.8b, #0
@@ -315,6 +341,11 @@ define <4 x i32> @sudot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: smlal2 v0.4s, v2.8h, v1.8h
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sudot:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v2.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sudot:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: sshll v3.8h, v1.8b, #0
@@ -357,6 +388,22 @@ define <4 x i32> @sudot_in_loop(ptr %p1, ptr %p2){
; CHECK-NOI8MM-NEXT: // %bb.2: // %end
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sudot_in_loop:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: mov x8, xzr
+; CHECK-DOT-I8MM-NEXT: .LBB9_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT: add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: cmp x8, #16
+; CHECK-DOT-I8MM-NEXT: b.ne .LBB9_1
+; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sudot_in_loop:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: movi v1.2d, #0000000000000000
@@ -419,6 +466,11 @@ define <2 x i32> @sudot_narrow(<2 x i32> %acc, <8 x i8> %u, <8 x i8> %s) #0{
; CHECK-NOI8MM-NEXT: add v0.2s, v1.2s, v0.2s
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sudot_narrow:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: usdot v0.2s, v2.8b, v1.8b
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sudot_narrow:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: sshll v1.8h, v1.8b, #0
@@ -461,6 +513,14 @@ define <4 x i64> @udot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: udot_8to64:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: udot_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
@@ -504,6 +564,14 @@ define <4 x i64> @sdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v4.4s
; CHECK-NODOT-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sdot_8to64:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sdot_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: movi v4.2d, #0000000000000000
@@ -553,6 +621,14 @@ define <4 x i64> @usdot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b){
; CHECK-NOI8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: usdot_8to64:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: usdot_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: ushll v4.8h, v2.8b, #0
@@ -610,6 +686,14 @@ define <4 x i64> @sudot_8to64(<4 x i64> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-NOI8MM-NEXT: smlal2 v1.2d, v2.4s, v3.4s
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sudot_8to64:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: usdot v4.4s, v3.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sudot_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: sshll v4.8h, v2.8b, #0
@@ -684,6 +768,22 @@ define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
; CHECK-NODOT-NEXT: // %bb.2: // %end
; CHECK-NODOT-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_in_loop:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: movi v2.16b, #1
+; CHECK-DOT-I8MM-NEXT: mov x8, xzr
+; CHECK-DOT-I8MM-NEXT: .LBB16_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT: ldr q3, [x0, x8]
+; CHECK-DOT-I8MM-NEXT: mov v0.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT: add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT: cmp x8, #16
+; CHECK-DOT-I8MM-NEXT: udot v1.4s, v3.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: b.ne .LBB16_1
+; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: udot_no_bin_op_in_loop:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: adrp x8, .LCPI16_0
@@ -854,6 +954,15 @@ define <4 x i64> @udot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-NEXT: uaddw2 v0.2d, v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_8to64:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v3.16b, #1
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: udot v4.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: udot_no_bin_op_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: ushll v3.8h, v2.8b, #0
@@ -913,6 +1022,15 @@ define <4 x i64> @sdot_no_bin_op_8to64(<4 x i64> %acc, <16 x i8> %a){
; CHECK-NODOT-NEXT: saddw2 v0.2d, v0.2d, v3.4s
; CHECK-NODOT-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: sdot_no_bin_op_8to64:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v3.16b, #1
+; CHECK-DOT-I8MM-NEXT: movi v4.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: sdot v4.4s, v2.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: saddw2 v1.2d, v1.2d, v4.4s
+; CHECK-DOT-I8MM-NEXT: saddw v0.2d, v0.2d, v4.2s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: sdot_no_bin_op_8to64:
; CHECK-NEWLOWERING-I8MM: // %bb.0:
; CHECK-NEWLOWERING-I8MM-NEXT: sshll v3.8h, v2.8b, #0
@@ -1101,6 +1219,25 @@ define <4 x i32> @usdot_multiple_zext_users(ptr %p1, ptr %p2, ptr %p3) {
; CHECK-NOI8MM-NEXT: add v0.4s, v1.4s, v0.4s
; CHECK-NOI8MM-NEXT: ret
;
+; CHECK-DOT-I8MM-LABEL: usdot_multiple_zext_users:
+; CHECK-DOT-I8MM: // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT: movi v0.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: mov x8, xzr
+; CHECK-DOT-I8MM-NEXT: .LBB28_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT: ldr q2, [x0, x8]
+; CHECK-DOT-I8MM-NEXT: ldr q3, [x1, x8]
+; CHECK-DOT-I8MM-NEXT: ldr q4, [x2, x8]
+; CHECK-DOT-I8MM-NEXT: add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT: usdot v0.4s, v4.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: usdot v1.4s, v4.16b, v3.16b
+; CHECK-DOT-I8MM-NEXT: cmp x8, #1024
+; CHECK-DOT-I8MM-NEXT: b.ne .LBB28_1
+; CHECK-DOT-I8MM-NEXT: // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT: add v0.4s, v1.4s, v0.4s
+; CHECK-DOT-I8MM-NEXT: ret
+;
; CHECK-NEWLOWERING-I8MM-LABEL: usdot_multiple_zext_users:
; CHECK-NEWLOWERING-I8MM: // %bb.0: // %entry
; CHECK-NEWLOWERING-I8MM-NEXT: movi v0.2d, #0000000000000000
>From d6d1f56006389d8bd1a0a9bb9ad0156279de5a61 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Wed, 21 May 2025 14:59:55 +0100
Subject: [PATCH 5/6] Update test
---
.../AArch64/sve-partial-reduce-dot-product.ll | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
index 5bc9a101b1e44..809a45045b0db 100644
--- a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
@@ -202,10 +202,10 @@ define <vscale x 4 x i64> @udot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8
; CHECK-NEWLOWERING-SVE: // %bb.0: // %entry
; CHECK-NEWLOWERING-SVE-NEXT: movi v4.2d, #0000000000000000
; CHECK-NEWLOWERING-SVE-NEXT: udot z4.s, z2.b, z3.b
-; CHECK-NEWLOWERING-SVE-NEXT: uunpkhi z2.d, z4.s
-; CHECK-NEWLOWERING-SVE-NEXT: uunpklo z3.d, z4.s
-; CHECK-NEWLOWERING-SVE-NEXT: add z2.d, z3.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT: uunpklo z2.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT: uunpkhi z3.d, z4.s
; CHECK-NEWLOWERING-SVE-NEXT: add z0.d, z0.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT: add z0.d, z0.d, z3.d
; CHECK-NEWLOWERING-SVE-NEXT: ret
;
; CHECK-NEWLOWERING-SVE2-LABEL: udot_8to64:
@@ -247,10 +247,10 @@ define <vscale x 4 x i64> @sdot_8to64(<vscale x 4 x i64> %acc, <vscale x 16 x i8
; CHECK-NEWLOWERING-SVE: // %bb.0: // %entry
; CHECK-NEWLOWERING-SVE-NEXT: movi v4.2d, #0000000000000000
; CHECK-NEWLOWERING-SVE-NEXT: sdot z4.s, z2.b, z3.b
-; CHECK-NEWLOWERING-SVE-NEXT: sunpkhi z2.d, z4.s
-; CHECK-NEWLOWERING-SVE-NEXT: sunpklo z3.d, z4.s
-; CHECK-NEWLOWERING-SVE-NEXT: add z2.d, z3.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT: sunpklo z2.d, z4.s
+; CHECK-NEWLOWERING-SVE-NEXT: sunpkhi z3.d, z4.s
; CHECK-NEWLOWERING-SVE-NEXT: add z0.d, z0.d, z2.d
+; CHECK-NEWLOWERING-SVE-NEXT: add z0.d, z0.d, z3.d
; CHECK-NEWLOWERING-SVE-NEXT: ret
;
; CHECK-NEWLOWERING-SVE2-LABEL: sdot_8to64:
>From 33088b427824cc96ca84bfce9f721036381c3483 Mon Sep 17 00:00:00 2001
From: Nick Guy <nicholas.guy at arm.com>
Date: Thu, 22 May 2025 11:50:56 +0100
Subject: [PATCH 6/6] Address comments
---
.../Target/AArch64/AArch64ISelLowering.cpp | 29 ++++++++++---------
.../neon-partial-reduce-dot-product.ll | 1 -
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9660b8331c3b8..ab0990f5937d0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1451,8 +1451,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
setOperationAction(ISD::ADD, VT, Custom);
// FADDP custom lowering
- for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
+ for (MVT VT : {MVT::v16f16, MVT::v8f32, MVT::v4f64})
setOperationAction(ISD::FADD, VT, Custom);
+
+ if (EnablePartialReduceNodes && Subtarget->hasDotProd()) {
+ setPartialReduceMLAAction(MVT::v2i64, MVT::v8i16, Legal);
+ setPartialReduceMLAAction(MVT::v4i32, MVT::v16i8, Legal);
+ setPartialReduceMLAAction(MVT::v2i32, MVT::v8i8, Legal);
+ setPartialReduceMLAAction(MVT::v2i64, MVT::v16i8, Custom);
+ }
+
} else /* !isNeonAvailable */ {
for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
@@ -1872,15 +1880,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setPartialReduceMLAAction(MVT::nxv2i64, MVT::nxv16i8, Custom);
}
- if (EnablePartialReduceNodes && Subtarget->hasNEON() &&
- Subtarget->hasDotProd()) {
- setPartialReduceMLAAction(MVT::v2i64, MVT::v8i16, Legal);
- setPartialReduceMLAAction(MVT::v4i32, MVT::v16i8, Legal);
- setPartialReduceMLAAction(MVT::v4i32, MVT::v16i8, Legal);
- setPartialReduceMLAAction(MVT::v2i32, MVT::v8i8, Legal);
- setPartialReduceMLAAction(MVT::v2i64, MVT::v16i8, Custom);
- }
-
// Handle operations that are only available in non-streaming SVE mode.
if (Subtarget->isSVEAvailable()) {
for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64,
@@ -29544,10 +29543,12 @@ SDValue
AArch64TargetLowering::LowerPARTIAL_REDUCE_MLA(SDValue Op,
SelectionDAG &DAG) const {
bool Scalable = Op.getValueType().isScalableVector();
- if (Scalable && !Subtarget->isSVEorStreamingSVEAvailable())
- return SDValue();
- if (!Scalable && (!Subtarget->isNeonAvailable() || !Subtarget->hasDotProd()))
- return SDValue();
+
+ assert((!Scalable || Subtarget->isSVEorStreamingSVEAvailable()) &&
+ "SVE or StreamingSVE must be available when using scalable vectors.");
+ assert(
+ (Scalable || (Subtarget->isNeonAvailable() || Subtarget->hasDotProd())) &&
+ "Neon or dotprod must be available when using fixed-width vectors.");
SDLoc DL(Op);
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 1f8c7f0fc74f6..fe5d6ca565b6d 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -3,7 +3,6 @@
; RUN: llc -mtriple aarch64 -mattr=+neon < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOI8MM,CHECK-NODOT
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM,CHECK-DOT-I8MM
; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod,+i8mm -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-I8MM,CHECK-NEWLOWERING-I8MM
-; RUN: llc -mtriple aarch64 -mattr=+neon,+dotprod -aarch64-enable-partial-reduce-nodes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-DOT,CHECK-NOI8MM,CHECK-NEWLOWERING-NOI8MM
define <4 x i32> @udot(<4 x i32> %acc, <16 x i8> %u, <16 x i8> %s) {
; CHECK-DOT-LABEL: udot:
More information about the llvm-commits
mailing list